You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by br...@apache.org on 2009/02/20 05:54:00 UTC
svn commit: r746136 - in /maven/components/branches/maven-2.1.x: ./
apache-maven/ maven-artifact-manager/
maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/
maven-core/src/main/java/org/apache/maven/cli/ maven-core/src/test/java/o...
Author: brett
Date: Fri Feb 20 04:53:53 2009
New Revision: 746136
URL: http://svn.apache.org/viewvc?rev=746136&view=rev
Log:
[MNG-3379] Parallel resolution of artifacts
Submitted by: Don Brown
Modified:
maven/components/branches/maven-2.1.x/ (props changed)
maven/components/branches/maven-2.1.x/apache-maven/ (props changed)
maven/components/branches/maven-2.1.x/apache-maven/pom.xml
maven/components/branches/maven-2.1.x/maven-artifact-manager/pom.xml
maven/components/branches/maven-2.1.x/maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/AbstractConsoleDownloadMonitor.java
maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/BatchModeDownloadMonitor.java
maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/ConsoleDownloadMonitor.java
maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/AbstractConsoleDownloadMonitorTest.java
maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/ConsoleDownloadMonitorTest.java
maven/components/branches/maven-2.1.x/pom.xml
Propchange: maven/components/branches/maven-2.1.x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 04:53:53 2009
@@ -3,3 +3,4 @@
/maven/components/branches/maven-2.1.0-M1-RC:690315,691793-694304
/maven/components/branches/maven-2.1.0-RC:688883,689695,689976,689990
/maven/components/trunk:572229,720001,721902,726845,729292
+/maven/sandbox/branches/maven/MNG-3379:678167
Propchange: maven/components/branches/maven-2.1.x/apache-maven/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 20 04:53:53 2009
@@ -4,3 +4,4 @@
/maven/components/branches/maven-2.1.0-RC/apache-maven:688883,689695,689976,689990
/maven/components/trunk/apache-maven:720001,721902,726845,727688
/maven/components/trunk/maven-distribution:727688
+/maven/sandbox/branches/maven/MNG-3379/apache-maven:678167
Modified: maven/components/branches/maven-2.1.x/apache-maven/pom.xml
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/apache-maven/pom.xml?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/apache-maven/pom.xml (original)
+++ maven/components/branches/maven-2.1.x/apache-maven/pom.xml Fri Feb 20 04:53:53 2009
@@ -89,6 +89,13 @@
<relocation>
<pattern>org.apache.commons.logging</pattern>
</relocation>
+
+ <!-- For maven-artifact-manager -->
+ <relocation>
+ <pattern>edu.emory.mathcs.backport</pattern>
+ </relocation>
+
+ <!-- For jsch wagon -->
<relocation>
<pattern>org.apache.commons.codec</pattern>
</relocation>
Modified: maven/components/branches/maven-2.1.x/maven-artifact-manager/pom.xml
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-artifact-manager/pom.xml?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-artifact-manager/pom.xml (original)
+++ maven/components/branches/maven-2.1.x/maven-artifact-manager/pom.xml Fri Feb 20 04:53:53 2009
@@ -52,6 +52,11 @@
<artifactId>wagon-provider-api</artifactId>
</dependency>
<dependency>
+ <groupId>backport-util-concurrent</groupId>
+ <artifactId>backport-util-concurrent</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
<groupId>easymock</groupId>
<artifactId>easymock</artifactId>
<version>1.2_Java1.3</version>
Modified: maven/components/branches/maven-2.1.x/maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java (original)
+++ maven/components/branches/maven-2.1.x/maven-artifact-manager/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java Fri Feb 20 04:53:53 2009
@@ -39,15 +39,14 @@
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.HashMap;
+import edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor;
+import edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue;
+import edu.emory.mathcs.backport.java.util.concurrent.TimeUnit;
+import edu.emory.mathcs.backport.java.util.concurrent.CountDownLatch;
+
public class DefaultArtifactResolver
extends AbstractLogEnabled
implements ArtifactResolver
@@ -63,6 +62,13 @@
protected ArtifactFactory artifactFactory;
private ArtifactCollector artifactCollector;
+ private final ThreadPoolExecutor resolveArtifactPool;
+
+ public DefaultArtifactResolver()
+ {
+ super();
+ resolveArtifactPool = new ThreadPoolExecutor(3, 5, 3, TimeUnit.SECONDS, new LinkedBlockingQueue());
+ }
// ----------------------------------------------------------------------
// Implementation
@@ -301,23 +307,37 @@
localRepository, remoteRepositories, source, filter,
listeners );
- List resolvedArtifacts = new ArrayList();
- List missingArtifacts = new ArrayList();
+ List resolvedArtifacts = Collections.synchronizedList(new ArrayList());
+ List missingArtifacts = Collections.synchronizedList(new ArrayList());
+ CountDownLatch latch = new CountDownLatch(artifactResolutionResult.getArtifactResolutionNodes().size());
+ Map nodesByGroupId = new HashMap();
for ( Iterator i = artifactResolutionResult.getArtifactResolutionNodes().iterator(); i.hasNext(); )
{
ResolutionNode node = (ResolutionNode) i.next();
- try
+ List nodes = (List) nodesByGroupId.get(node.getArtifact().getGroupId());
+ if (nodes == null)
{
- resolve( node.getArtifact(), node.getRemoteRepositories(), localRepository );
- resolvedArtifacts.add( node.getArtifact() );
+ nodes = new ArrayList();
+ nodesByGroupId.put(node.getArtifact().getGroupId(), nodes);
}
- catch ( ArtifactNotFoundException anfe )
- {
- getLogger().debug( anfe.getMessage(), anfe );
+ nodes.add(node);
+ }
- missingArtifacts.add( node.getArtifact() );
+ try {
+ for (Iterator i = nodesByGroupId.values().iterator(); i.hasNext(); )
+ {
+ List nodes = (List) i.next();
+ resolveArtifactPool.execute(new ResolveArtifactTask(resolveArtifactPool, latch, nodes, localRepository, resolvedArtifacts, missingArtifacts));
}
- }
+ latch.await();
+ } catch (InterruptedException e) {
+ throw new ArtifactResolutionException("Resolution interrupted", null, e);
+ } catch (RuntimeException ex) {
+ if (ex.getCause() instanceof ArtifactResolutionException)
+ throw (ArtifactResolutionException) ex.getCause();
+ else
+ throw ex;
+ }
if ( missingArtifacts.size() > 0 )
{
@@ -357,4 +377,51 @@
remoteRepositories, source, null, listeners );
}
+ private class ResolveArtifactTask implements Runnable {
+ private List nodes;
+ private ArtifactRepository localRepository;
+ private List resolvedArtifacts;
+ private List missingArtifacts;
+ private CountDownLatch latch;
+ private ThreadPoolExecutor pool;
+
+ public ResolveArtifactTask(ThreadPoolExecutor pool, CountDownLatch latch, List nodes, ArtifactRepository localRepository, List resolvedArtifacts, List missingArtifacts) {
+ this.nodes = nodes;
+ this.localRepository = localRepository;
+ this.resolvedArtifacts = resolvedArtifacts;
+ this.missingArtifacts = missingArtifacts;
+ this.latch = latch;
+ this.pool = pool;
+ }
+
+ public void run() {
+ //getLogger().info("Size of nodes: "+nodes.size()+" on thread: "+Thread.currentThread().getId());
+ Iterator i = nodes.iterator();
+ ResolutionNode node = (ResolutionNode) i.next();
+ i.remove();
+ try {
+ resolveArtifact(node);
+ if (i.hasNext())
+ pool.execute(new ResolveArtifactTask(pool, latch, nodes, localRepository, resolvedArtifacts, missingArtifacts));
+ } catch (ArtifactResolutionException e) {
+ throw new RuntimeException(e);
+ }
+ latch.countDown();
+ }
+
+ private void resolveArtifact(ResolutionNode node) throws ArtifactResolutionException {
+ try
+ {
+ resolve( node.getArtifact(), node.getRemoteRepositories(), localRepository );
+ resolvedArtifacts.add( node.getArtifact() );
+ }
+ catch ( ArtifactNotFoundException anfe )
+ {
+ getLogger().debug( anfe.getMessage(), anfe );
+
+ missingArtifacts.add( node.getArtifact() );
+ }
+ }
+ }
+
}
Modified: maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/AbstractConsoleDownloadMonitor.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/AbstractConsoleDownloadMonitor.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/AbstractConsoleDownloadMonitor.java (original)
+++ maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/AbstractConsoleDownloadMonitor.java Fri Feb 20 04:53:53 2009
@@ -25,6 +25,8 @@
import org.codehaus.plexus.logging.AbstractLogEnabled;
import org.codehaus.plexus.logging.Logger;
+import java.io.PrintStream;
+
/**
* Abstract console download progress meter.
*
@@ -38,6 +40,8 @@
{
private Logger logger;
+ PrintStream out = System.out;
+
public AbstractConsoleDownloadMonitor()
{
}
@@ -54,7 +58,7 @@
String url = transferEvent.getWagon().getRepository().getUrl();
// TODO: can't use getLogger() because this isn't currently instantiated as a component
- System.out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
+ out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
}
/**
@@ -80,7 +84,9 @@
{
String type = ( transferEvent.getRequestType() == TransferEvent.REQUEST_PUT ? "uploaded" : "downloaded" );
String l = contentLength >= 1024 ? ( contentLength / 1024 ) + "K" : contentLength + "b";
- System.out.println( l + " " + type );
+ String name = transferEvent.getResource().getName();
+ name = name.substring( name.lastIndexOf( '/' ) + 1, name.length() );
+ out.println( l + " " + type + " (" + name + ")" );
}
}
Modified: maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/BatchModeDownloadMonitor.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/BatchModeDownloadMonitor.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/BatchModeDownloadMonitor.java (original)
+++ maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/BatchModeDownloadMonitor.java Fri Feb 20 04:53:53 2009
@@ -46,6 +46,6 @@
String url = transferEvent.getWagon().getRepository().getUrl();
- System.out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
+ out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
}
}
Modified: maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/ConsoleDownloadMonitor.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/ConsoleDownloadMonitor.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/ConsoleDownloadMonitor.java (original)
+++ maven/components/branches/maven-2.1.x/maven-core/src/main/java/org/apache/maven/cli/ConsoleDownloadMonitor.java Fri Feb 20 04:53:53 2009
@@ -20,11 +20,14 @@
*/
import org.apache.maven.wagon.WagonConstants;
+import org.apache.maven.wagon.resource.Resource;
import org.apache.maven.wagon.events.TransferEvent;
import org.codehaus.plexus.logging.Logger;
+import java.util.*;
+
/**
- * Console download progress meter.
+ * Console download progress meter. Properly handles multiple downloads simultaneously.
*
* @author <a href="mailto:brett@apache.org">Brett Porter</a>
* @version $Id$
@@ -32,15 +35,18 @@
public class ConsoleDownloadMonitor
extends AbstractConsoleDownloadMonitor
{
- private long complete;
+ private Map/*<Resource,Integer>*/ downloads;
public ConsoleDownloadMonitor( Logger logger )
{
super( logger );
+
+ downloads = new LinkedHashMap();
}
public ConsoleDownloadMonitor()
{
+ downloads = new LinkedHashMap();
}
public void transferInitiated( TransferEvent transferEvent )
@@ -49,9 +55,8 @@
String url = transferEvent.getWagon().getRepository().getUrl();
- System.out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
+ out.println( message + ": " + url + "/" + transferEvent.getResource().getName() );
- complete = 0;
}
public void transferStarted( TransferEvent transferEvent )
@@ -59,20 +64,49 @@
// This space left intentionally blank
}
- public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
+ public synchronized void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
+ {
+ Resource resource = transferEvent.getResource();
+ if (!downloads.containsKey(resource))
+ {
+ downloads.put(resource, new Long(length));
+ } else
+ {
+ Long complete = (Long) downloads.get(resource);
+ complete = new Long(complete.longValue() + length);
+ downloads.put(resource, complete);
+ }
+
+ for (Iterator i = downloads.entrySet().iterator(); i.hasNext(); )
+ {
+ Map.Entry entry = (Map.Entry) i.next();
+ Long complete = (Long)entry.getValue();
+ String status = getDownloadStatusForResource(complete.longValue(), ((Resource)entry.getKey()).getContentLength());
+ out.print(status);
+ if (i.hasNext())
+ {
+ out.print(" ");
+ }
+ }
+ out.print("\r");
+ }
+
+ String getDownloadStatusForResource(long progress, long total)
{
- long total = transferEvent.getResource().getContentLength();
- complete += length;
if ( total >= 1024 )
{
- System.out.print(
- ( complete / 1024 ) + "/" + ( total == WagonConstants.UNKNOWN_LENGTH ? "?" : ( total / 1024 ) + "K" ) +
- "\r" );
+ return (progress / 1024 ) + "/" + ( total == WagonConstants.UNKNOWN_LENGTH ? "?" : ( total / 1024 ) + "K");
}
else
{
- System.out.print( complete + "/" + ( total == WagonConstants.UNKNOWN_LENGTH ? "?" : total + "b" ) + "\r" );
+ return progress + "/" + ( total == WagonConstants.UNKNOWN_LENGTH ? "?" : total + "b" );
}
}
+
+ public synchronized void transferCompleted( TransferEvent transferEvent )
+ {
+ super.transferCompleted(transferEvent);
+ downloads.remove(transferEvent.getResource());
+ }
}
Modified: maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/AbstractConsoleDownloadMonitorTest.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/AbstractConsoleDownloadMonitorTest.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/AbstractConsoleDownloadMonitorTest.java (original)
+++ maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/AbstractConsoleDownloadMonitorTest.java Fri Feb 20 04:53:53 2009
@@ -38,7 +38,7 @@
extends TestCase
{
- private AbstractConsoleDownloadMonitor monitor;
+ protected AbstractConsoleDownloadMonitor monitor;
public AbstractConsoleDownloadMonitorTest()
{
@@ -92,14 +92,25 @@
monitor.debug( "msg" );
}
- private class TransferEventMock
+ static class TransferEventMock
extends TransferEvent
{
+
public TransferEventMock()
throws ConnectionException, AuthenticationException
{
super( new FileWagon(), new Resource(), TransferEvent.TRANSFER_INITIATED, TransferEvent.REQUEST_GET );
getResource().setContentLength( 100000 );
+ getResource().setName( "foo.bar" );
+ Repository repository = new Repository();
+ getWagon().connect( repository );
+ }
+
+ public TransferEventMock(Resource resource, int length)
+ throws ConnectionException, AuthenticationException
+ {
+ super( new FileWagon(), resource, TransferEvent.TRANSFER_INITIATED, TransferEvent.REQUEST_GET );
+ getResource().setContentLength(length);
Repository repository = new Repository();
getWagon().connect( repository );
}
@@ -109,6 +120,7 @@
{
super( new FileWagon(), new Resource(), exception, TransferEvent.REQUEST_GET );
getResource().setContentLength( 100000 );
+ getResource().setName( "foo.bar" );
Repository repository = new Repository();
getWagon().connect( repository );
}
Modified: maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/ConsoleDownloadMonitorTest.java
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/ConsoleDownloadMonitorTest.java?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/ConsoleDownloadMonitorTest.java (original)
+++ maven/components/branches/maven-2.1.x/maven-core/src/test/java/org/apache/maven/cli/ConsoleDownloadMonitorTest.java Fri Feb 20 04:53:53 2009
@@ -1,5 +1,12 @@
package org.apache.maven.cli;
+import org.apache.maven.wagon.resource.Resource;
+import org.apache.maven.wagon.WagonConstants;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintWriter;
+import java.io.PrintStream;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -28,11 +35,48 @@
public class ConsoleDownloadMonitorTest
extends AbstractConsoleDownloadMonitorTest
{
-
+ ByteArrayOutputStream bout;
protected void setUp()
throws Exception
{
super.setMonitor( new ConsoleDownloadMonitor() );
super.setUp();
+ bout = new ByteArrayOutputStream();
+ monitor.out = new PrintStream(bout);
+ }
+
+ public void testTransferProgress()
+ throws Exception
+ {
+ byte[] buffer = new byte[1024];
+ monitor.transferProgress( new TransferEventMock(new Resource(), 10000), buffer, 1024 );
+ assertEquals("1/9K\r", new String(bout.toByteArray()));
+ }
+
+ public void testTransferProgressTwoFiles()
+ throws Exception
+ {
+ byte[] buffer = new byte[2048];
+ monitor.transferProgress( new TransferEventMock(new Resource("foo"), 10000), buffer, 1024 );
+ assertEquals("1/9K\r", new String(bout.toByteArray()));
+ bout.reset();
+ monitor.transferProgress( new TransferEventMock(new Resource("bar"), 10000), buffer, 2048 );
+ assertEquals("1/9K 2/9K\r", new String(bout.toByteArray()));
+ bout.reset();
+ monitor.transferProgress( new TransferEventMock(new Resource("bar"), 10000), buffer, 2048 );
+ assertEquals("1/9K 4/9K\r", new String(bout.toByteArray()));
+ bout.reset();
+ monitor.transferProgress( new TransferEventMock(new Resource("foo"), 10000), buffer, 2048 );
+ assertEquals("3/9K 4/9K\r", new String(bout.toByteArray()));
+ }
+
+ public void testGetDownloadStatusForResource()
+ {
+ ConsoleDownloadMonitor cm = (ConsoleDownloadMonitor)monitor;
+ assertEquals("200/400b", cm.getDownloadStatusForResource(200, 400));
+ assertEquals("1/2K", cm.getDownloadStatusForResource(1024, 2048));
+ assertEquals("0/2K", cm.getDownloadStatusForResource(10, 2048));
+ assertEquals("10/?", cm.getDownloadStatusForResource(10, WagonConstants.UNKNOWN_LENGTH));
+ assertEquals("1024/?", cm.getDownloadStatusForResource(1024, WagonConstants.UNKNOWN_LENGTH));
}
}
Modified: maven/components/branches/maven-2.1.x/pom.xml
URL: http://svn.apache.org/viewvc/maven/components/branches/maven-2.1.x/pom.xml?rev=746136&r1=746135&r2=746136&view=diff
==============================================================================
--- maven/components/branches/maven-2.1.x/pom.xml (original)
+++ maven/components/branches/maven-2.1.x/pom.xml Fri Feb 20 04:53:53 2009
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.maven</groupId>
<artifactId>maven-parent</artifactId>
- <version>11-SNAPSHOT</version>
+ <version>11</version>
<relativePath>../pom/maven/pom.xml</relativePath>
</parent>