You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by df...@apache.org on 2009/11/06 22:46:48 UTC
svn commit: r833566 - in
/maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven:
execution/MavenSession.java lifecycle/DefaultLifecycleExecutor.java
Author: dfabulich
Date: Fri Nov 6 21:46:48 2009
New Revision: 833566
URL: http://svn.apache.org/viewvc?rev=833566&view=rev
Log:
Adding experimental multithreading support. Naive implementation. Not guaranteed to work. Builder beware. You'd be crazy to use this... -Dmaven.threads=4
Modified:
maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/execution/MavenSession.java
maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java
Modified: maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/execution/MavenSession.java
URL: http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/execution/MavenSession.java?rev=833566&r1=833565&r2=833566&view=diff
==============================================================================
--- maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/execution/MavenSession.java (original)
+++ maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/execution/MavenSession.java Fri Nov 6 21:46:48 2009
@@ -43,7 +43,7 @@
* @author Jason van Zyl
* @version $Id$
*/
-public class MavenSession
+public class MavenSession implements Cloneable
{
private PlexusContainer container;
@@ -318,6 +318,11 @@
{
this.projectDependencyGraph = projectDependencyGraph;
}
+
+ public ProjectDependencyGraph getProjectDependencyGraph()
+ {
+ return projectDependencyGraph;
+ }
public String getReactorFailureBehavior()
{
@@ -361,4 +366,27 @@
return request.getStartTime();
}
+ @Override
+ public MavenSession clone()
+ {
+ try
+ {
+ return (MavenSession) super.clone();
+ }
+ catch ( CloneNotSupportedException e )
+ {
+ throw new RuntimeException("Bug", e);
+ }
+ }
+
+ public void merge( MavenSession session )
+ {
+ if ( session.blackListedProjects != null )
+ {
+ for ( String projectId : session.blackListedProjects )
+ {
+ blackListedProjects.add( projectId );
+ }
+ }
+ }
}
Modified: maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java
URL: http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java?rev=833566&r1=833565&r2=833566&view=diff
==============================================================================
--- maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java (original)
+++ maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java Fri Nov 6 21:46:48 2009
@@ -31,6 +31,12 @@
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import org.apache.maven.ProjectDependenciesResolver;
import org.apache.maven.artifact.Artifact;
@@ -48,10 +54,12 @@
import org.apache.maven.execution.MavenExecutionRequest;
import org.apache.maven.execution.MavenExecutionResult;
import org.apache.maven.execution.MavenSession;
+import org.apache.maven.execution.ProjectDependencyGraph;
import org.apache.maven.lifecycle.mapping.LifecycleMapping;
import org.apache.maven.model.Dependency;
import org.apache.maven.model.Plugin;
import org.apache.maven.model.PluginExecution;
+import org.apache.maven.plugin.BuildPluginManager;
import org.apache.maven.plugin.InvalidPluginDescriptorException;
import org.apache.maven.plugin.MojoExecution;
import org.apache.maven.plugin.MojoExecutionException;
@@ -59,7 +67,6 @@
import org.apache.maven.plugin.MojoNotFoundException;
import org.apache.maven.plugin.PluginConfigurationException;
import org.apache.maven.plugin.PluginDescriptorParsingException;
-import org.apache.maven.plugin.BuildPluginManager;
import org.apache.maven.plugin.PluginManagerException;
import org.apache.maven.plugin.PluginNotFoundException;
import org.apache.maven.plugin.PluginResolutionException;
@@ -252,116 +259,242 @@
}
ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
+
+ HashMap<MavenProject, ProjectBuild> projectBuildMap = new HashMap<MavenProject, ProjectBuild>();
+ for (ProjectBuild projectBuild : projectBuilds ) {
+ projectBuildMap.put( projectBuild.project, projectBuild );
+ }
- for ( ProjectBuild projectBuild : projectBuilds )
+ ProjectDependencyGraph pdg = session.getProjectDependencyGraph();
+
+ int threadCount = 1;
+ String threadCountProperty = (String) session.getUserProperties().get( "maven.threads.experimental" );
+ if (threadCountProperty != null) {
+ try {
+ threadCount = Integer.parseInt( threadCountProperty );
+ } catch (NumberFormatException e) {
+ logger.warn( "Couldn't parse thread count, will default to 1: " + threadCountProperty );
+ }
+ }
+ if ( logger.isDebugEnabled() )
{
- MavenProject currentProject = projectBuild.project;
-
- long buildStartTime = System.currentTimeMillis();
-
+ logger.debug( "Thread pool: " + threadCount );
+ }
+ Executor executor = Executors.newFixedThreadPool( threadCount );
+ CompletionService<IndividualProjectBuildResult> service = new ExecutorCompletionService<IndividualProjectBuildResult>( executor );
+ HashSet<Future<IndividualProjectBuildResult>> futures = new HashSet<Future<IndividualProjectBuildResult>>();
+
+ // schedule independent projects
+ for (ProjectBuild projectBuild : projectBuilds) {
+ if ( pdg.getUpstreamProjects( projectBuild.project, false ).size() == 0 ) {
+ if ( logger.isDebugEnabled() )
+ {
+ logger.debug( "Scheduling: " + projectBuild.project );
+ }
+ CallableBuilder cb = new CallableBuilder( result, projectBuild, projectIndex, oldContextClassLoader, session );
+ futures.add( service.submit( cb ) );
+ }
+ }
+
+ HashSet<MavenProject> finishedProjects = new HashSet<MavenProject>();
+
+ // for each finished project
+ for (int i = 0; i < projectBuilds.size(); i++) {
+ IndividualProjectBuildResult ipbr;
try
{
- session.setCurrentProject( currentProject );
+ ipbr = service.take().get();
+ }
+ catch ( Exception e )
+ {
+ break;
+ }
+ if (ipbr.shouldHaltBuild) break;
+ ProjectBuild projectBuild = ipbr.projectBuild;
+ MavenProject finishedProject = projectBuild.project;
+ finishedProjects.add( finishedProject );
+ // schedule dependent projects, if all of their requirements are met
+ for ( MavenProject dependentProject : pdg.getDownstreamProjects( finishedProject, false ) ) {
+ boolean allRequirementsMet = true;
+ for ( MavenProject requirement : pdg.getUpstreamProjects( dependentProject, false ) ) {
+ if (!finishedProjects.contains( requirement ) ) {
+ if ( logger.isDebugEnabled() )
+ {
+ logger.debug( "Not scheduling " + dependentProject + " because requirement not met: " + requirement);
+ }
+ allRequirementsMet = false;
+ break;
+ }
+ }
+ if (allRequirementsMet) {
+ ProjectBuild scheduledDependent = projectBuildMap.get( dependentProject );
+ if ( logger.isDebugEnabled() )
+ {
+ logger.debug( "Scheduling: " + dependentProject );
+ }
+ CallableBuilder cb = new CallableBuilder( result, scheduledDependent, projectIndex, oldContextClassLoader, session );
+ futures.add( service.submit( cb ) );
+ }
+ }
+ }
+
+ // cancel outstanding builds (if any)
+ for (Future<IndividualProjectBuildResult> future : futures) {
+ future.cancel( true /* mayInterruptIfRunning */ );
+ }
+
- if ( session.isBlackListed( currentProject ) )
- {
- fireEvent( session, null, LifecycleEventCatapult.PROJECT_SKIPPED );
+ fireEvent( session, null, LifecycleEventCatapult.SESSION_ENDED );
+ }
- continue;
- }
+ private class CallableBuilder implements Callable<IndividualProjectBuildResult> {
- fireEvent( session, null, LifecycleEventCatapult.PROJECT_STARTED );
+ final MavenExecutionResult result;
+ final ProjectBuild projectBuild;
+ final ProjectIndex projectIndex;
+ final ClassLoader oldContextClassLoader;
+ final MavenSession originalSession;
+
+ public CallableBuilder( MavenExecutionResult result, ProjectBuild projectBuild, ProjectIndex projectIndex,
+ ClassLoader oldContextClassLoader, MavenSession originalSession )
+ {
+ this.result = result;
+ this.projectBuild = projectBuild;
+ this.projectIndex = projectIndex;
+ this.oldContextClassLoader = oldContextClassLoader;
+ this.originalSession = originalSession;
+ }
+
+ public IndividualProjectBuildResult call()
+ throws Exception
+ {
+ boolean shouldHaltBuild = buildProject( result, projectBuild, projectIndex, oldContextClassLoader, originalSession );
+ return new IndividualProjectBuildResult( shouldHaltBuild, projectBuild );
+ }
+ }
+
+ private class IndividualProjectBuildResult {
+ public IndividualProjectBuildResult( boolean shouldHaltBuild, ProjectBuild projectBuild )
+ {
+ this.shouldHaltBuild = shouldHaltBuild;
+ this.projectBuild = projectBuild;
+ }
+ final boolean shouldHaltBuild;
+ final ProjectBuild projectBuild;
+
+ }
+
+ private boolean buildProject( MavenExecutionResult result,
+ ProjectBuild projectBuild, ProjectIndex projectIndex,
+ ClassLoader oldContextClassLoader, MavenSession originalSession )
+ {
+ MavenSession session = originalSession.clone();
+ MavenProject currentProject = projectBuild.project;
- ClassRealm projectRealm = currentProject.getClassRealm();
- if ( projectRealm != null )
- {
- Thread.currentThread().setContextClassLoader( projectRealm );
- }
+ long buildStartTime = System.currentTimeMillis();
- MavenExecutionPlan executionPlan =
- calculateExecutionPlan( session, currentProject, projectBuild.taskSegment );
+ try
+ {
+ session.setCurrentProject( currentProject );
- if ( logger.isDebugEnabled() )
- {
- debugProjectPlan( currentProject, executionPlan );
- }
+ if ( session.isBlackListed( currentProject ) )
+ {
+ fireEvent( session, null, LifecycleEventCatapult.PROJECT_SKIPPED );
- // TODO: once we have calculated the build plan then we should accurately be able to download
- // the project dependencies. Having it happen in the plugin manager is a tangled mess. We can optimize
- // this later by looking at the build plan. Would be better to just batch download everything required
- // by the reactor.
+ return false;
+ }
- List<MavenProject> projectsToResolve;
+ fireEvent( session, null, LifecycleEventCatapult.PROJECT_STARTED );
- if ( projectBuild.taskSegment.aggregating )
- {
- projectsToResolve = session.getProjects();
- }
- else
- {
- projectsToResolve = Collections.singletonList( currentProject );
- }
+ ClassRealm projectRealm = currentProject.getClassRealm();
+ if ( projectRealm != null )
+ {
+ Thread.currentThread().setContextClassLoader( projectRealm );
+ }
- for ( MavenProject project : projectsToResolve )
- {
- resolveProjectDependencies( project, executionPlan.getRequiredCollectionScopes(),
- executionPlan.getRequiredResolutionScopes(), session,
- projectBuild.taskSegment.aggregating );
- }
+ MavenExecutionPlan executionPlan =
+ calculateExecutionPlan( session, currentProject, projectBuild.taskSegment );
- DependencyContext dependencyContext =
- new DependencyContext( executionPlan, projectBuild.taskSegment.aggregating );
+ if ( logger.isDebugEnabled() )
+ {
+ debugProjectPlan( currentProject, executionPlan );
+ }
- for ( MojoExecution mojoExecution : executionPlan.getExecutions() )
- {
- execute( session, mojoExecution, projectIndex, dependencyContext );
- }
+ // TODO: once we have calculated the build plan then we should accurately be able to download
+ // the project dependencies. Having it happen in the plugin manager is a tangled mess. We can optimize
+ // this later by looking at the build plan. Would be better to just batch download everything required
+ // by the reactor.
- long buildEndTime = System.currentTimeMillis();
+ List<MavenProject> projectsToResolve;
- result.addBuildSummary( new BuildSuccess( currentProject, buildEndTime - buildStartTime ) );
+ if ( projectBuild.taskSegment.aggregating )
+ {
+ projectsToResolve = session.getProjects();
+ }
+ else
+ {
+ projectsToResolve = Collections.singletonList( currentProject );
+ }
- fireEvent( session, null, LifecycleEventCatapult.PROJECT_SUCCEEDED );
+ for ( MavenProject project : projectsToResolve )
+ {
+ resolveProjectDependencies( project, executionPlan.getRequiredCollectionScopes(),
+ executionPlan.getRequiredResolutionScopes(), session,
+ projectBuild.taskSegment.aggregating );
}
- catch ( Exception e )
+
+ DependencyContext dependencyContext =
+ new DependencyContext( executionPlan, projectBuild.taskSegment.aggregating );
+
+ for ( MojoExecution mojoExecution : executionPlan.getExecutions() )
{
- result.addException( e );
+ execute( session, mojoExecution, projectIndex, dependencyContext );
+ }
- long buildEndTime = System.currentTimeMillis();
+ long buildEndTime = System.currentTimeMillis();
- result.addBuildSummary( new BuildFailure( currentProject, buildEndTime - buildStartTime, e ) );
+ result.addBuildSummary( new BuildSuccess( currentProject, buildEndTime - buildStartTime ) );
- fireEvent( session, null, LifecycleEventCatapult.PROJECT_FAILED );
+ fireEvent( session, null, LifecycleEventCatapult.PROJECT_SUCCEEDED );
+ }
+ catch ( Exception e )
+ {
+ result.addException( e );
- if ( MavenExecutionRequest.REACTOR_FAIL_NEVER.equals( session.getReactorFailureBehavior() ) )
- {
- // continue the build
- }
- else if ( MavenExecutionRequest.REACTOR_FAIL_AT_END.equals( session.getReactorFailureBehavior() ) )
- {
- // continue the build but ban all projects that depend on the failed one
- session.blackList( currentProject );
- }
- else if ( MavenExecutionRequest.REACTOR_FAIL_FAST.equals( session.getReactorFailureBehavior() ) )
- {
- // abort the build
- break;
- }
- else
- {
- throw new IllegalArgumentException( "invalid reactor failure behavior "
- + session.getReactorFailureBehavior() );
- }
+ long buildEndTime = System.currentTimeMillis();
+
+ result.addBuildSummary( new BuildFailure( currentProject, buildEndTime - buildStartTime, e ) );
+
+ fireEvent( session, null, LifecycleEventCatapult.PROJECT_FAILED );
+
+ if ( MavenExecutionRequest.REACTOR_FAIL_NEVER.equals( session.getReactorFailureBehavior() ) )
+ {
+ // continue the build
}
- finally
+ else if ( MavenExecutionRequest.REACTOR_FAIL_AT_END.equals( session.getReactorFailureBehavior() ) )
{
- session.setCurrentProject( null );
-
- Thread.currentThread().setContextClassLoader( oldContextClassLoader );
+ // continue the build but ban all projects that depend on the failed one
+ session.blackList( currentProject );
+ }
+ else if ( MavenExecutionRequest.REACTOR_FAIL_FAST.equals( session.getReactorFailureBehavior() ) )
+ {
+ // abort the build
+ return true;
+ }
+ else
+ {
+ throw new IllegalArgumentException( "invalid reactor failure behavior "
+ + session.getReactorFailureBehavior() );
}
}
+ finally
+ {
+ session.setCurrentProject( null );
+ originalSession.merge( session );
- fireEvent( session, null, LifecycleEventCatapult.SESSION_ENDED );
+ Thread.currentThread().setContextClassLoader( oldContextClassLoader );
+ }
+ return false;
}
private void resolveProjectDependencies( MavenProject project, Collection<String> scopesToCollect,
Re: svn commit: r833566 - in /maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven:
execution/MavenSession.java lifecycle/DefaultLifecycleExecutor.java
Posted by Dan Fabulich <da...@fabulich.com>.
Benjamin Bentmann wrote:
> Benjamin Bentmann schrieb:
>
>> So I wouldn't like to see such experiments on trunk, given we are trying to
>> stabilize it. Hence I created a branch with your changes and reverted them
>> on trunk.
>
> In talking with Dan and Wendy on IRC I realized that my immediate reaction of
> reverting Dan's commit was not the proper way of dealing with this situation.
> So I would like to apologize for my rude action on this topic.
>
> There are apparently emotions on both sides of this story and I hope we can
> clear this.
I also would like to apologize for checking in broken code. I'll attempt
to fix my mistake today.
Onward! :-)
-Dan
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org
Re: svn commit: r833566 - in /maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven:
execution/MavenSession.java lifecycle/DefaultLifecycleExecutor.java
Posted by Benjamin Bentmann <be...@udo.edu>.
Benjamin Bentmann schrieb:
> So I wouldn't like to see such experiments on trunk, given we are trying
> to stabilize it. Hence I created a branch with your changes and reverted
> them on trunk.
In talking with Dan and Wendy on IRC I realized that my immediate
reaction of reverting Dan's commit was not the proper way of dealing
with this situation. So I would like to apologize for my rude action on
this topic.
There are apparently emotions on both sides of this story and I hope we
can clear this.
Benjamin
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org
Re: svn commit: r833566 - in /maven/maven-3/trunk/maven-core/src/main/java/org/apache/maven:
execution/MavenSession.java lifecycle/DefaultLifecycleExecutor.java
Posted by Benjamin Bentmann <be...@udo.edu>.
Hi Dan,
> Author: dfabulich
> Date: Fri Nov 6 21:46:48 2009
> New Revision: 833566
>
> URL: http://svn.apache.org/viewvc?rev=833566&view=rev
> Log:
> Adding experimental multithreading support. Naive implementation. Not guaranteed to work. Builder beware. You'd be crazy to use this... -Dmaven.threads=4
Please be more careful what you commit to trunk. Seeing words like
"experimental multithreading", "Not guaranteed to work" and "You'd be
crazy to use this" make me feel let's say uneasy. This significant (and
badly formatted) code change went in without any JIRA ticket, without
any recent proposal/discussion, without any kind of tests and eventually
breaks existing core ITs [0, 1].
I have no objections to you working on parallel builds but
multi-threading is something humans rarely get right upon first try as
we already know from the introduction of multi-threaded artifact
downloads and again witnessed by the failing ITs.
So I wouldn't like to see such experiments on trunk, given we are trying
to stabilize it. Hence I created a branch with your changes and reverted
them on trunk.
Benjamin
[0]
https://grid.sonatype.org/ci/job/maven-3.0.x-ITs/jdk=1.5,label=ubuntu/121/console
[1]
https://grid.sonatype.org/ci/job/maven-3.0.x-ITs/jdk=1.5,label=windows/121/console
[2] http://maven.apache.org/developers/conventions/code.html
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org