You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2022/10/26 06:35:18 UTC
[maven-resolver] branch master updated: [MRESOLVER-7] Download POMs in parallel in DF collector (#178)
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
The following commit(s) were added to refs/heads/master by this push:
new 5ce73125 [MRESOLVER-7] Download POMs in parallel in DF collector (#178)
5ce73125 is described below
commit 5ce731254e0b01f340d05ef1caddfff612bcbadf
Author: Eric Cai <we...@ebay.com>
AuthorDate: Wed Oct 26 14:35:13 2022 +0800
[MRESOLVER-7] Download POMs in parallel in DF collector (#178)
Parallel POM downloading in DF collector.
- only parallelize the VersionRange and ArtifactDesciptor resolution part (poms.xml & maven-metadata.xml downloading in parallel) as the poms downloading or maven-metadata.xml is the most slow part.
- Pure dependency resolution logic and the [skip logic](https://github.com/apache/maven-resolver/pull/158) are still run in sequence.
---
https://issues.apache.org/jira/browse/MRESOLVER-7
---
.../impl/collect/bf/BfDependencyCollector.java | 284 +++++++++++++++++----
.../collect/bf/DependencyProcessingContext.java | 17 +-
2 files changed, 252 insertions(+), 49 deletions(-)
diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java
index c82eed94..6145c595 100644
--- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java
+++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java
@@ -26,12 +26,28 @@ import javax.inject.Singleton;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import java.util.Queue;
-
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.concurrent.ConcurrentUtils;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.RequestTrace;
import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.artifact.ArtifactType;
+import org.eclipse.aether.artifact.DefaultArtifact;
import org.eclipse.aether.collection.CollectRequest;
import org.eclipse.aether.collection.DependencyManager;
import org.eclipse.aether.collection.DependencySelector;
@@ -53,12 +69,15 @@ import org.eclipse.aether.resolution.ArtifactDescriptorException;
import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
import org.eclipse.aether.resolution.ArtifactDescriptorResult;
import org.eclipse.aether.resolution.VersionRangeRequest;
-import org.eclipse.aether.resolution.VersionRangeResolutionException;
import org.eclipse.aether.resolution.VersionRangeResult;
import org.eclipse.aether.spi.locator.Service;
import org.eclipse.aether.util.ConfigUtils;
+import org.eclipse.aether.util.artifact.ArtifactIdUtils;
+import org.eclipse.aether.util.concurrency.WorkerThreadFactory;
import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
import org.eclipse.aether.version.Version;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
@@ -128,7 +147,8 @@ public class BfDependencyCollector
Args args =
new Args( session, pool, context, versionContext, request,
useSkip ? DependencyResolutionSkipper.defaultSkipper()
- : DependencyResolutionSkipper.neverSkipper() );
+ : DependencyResolutionSkipper.neverSkipper(),
+ new ParallelDescriptorResolver( session ) );
DependencySelector rootDepSelector = session.getDependencySelector() != null
? session.getDependencySelector().deriveChildSelector( context ) : null;
@@ -142,77 +162,65 @@ public class BfDependencyCollector
List<DependencyNode> parents = Collections.singletonList( node );
for ( Dependency dependency : dependencies )
{
- args.dependencyProcessingQueue.add(
+ RequestTrace childTrace = collectStepTrace( trace, args.request.getRequestContext(), parents,
+ dependency );
+ DependencyProcessingContext processingContext =
new DependencyProcessingContext( rootDepSelector, rootDepManager, rootDepTraverser,
- rootVerFilter, repositories, managedDependencies, parents,
- dependency ) );
+ rootVerFilter, childTrace, repositories, managedDependencies, parents, dependency,
+ PremanagedDependency.create( rootDepManager, dependency,
+ false, args.premanagedState ) );
+ if ( !filter( processingContext ) )
+ {
+ processingContext.withDependency( processingContext.premanagedDependency.getManagedDependency() );
+ resolveArtifactDescriptorAsync( args, processingContext, results );
+ args.dependencyProcessingQueue.add( processingContext );
+ }
}
while ( !args.dependencyProcessingQueue.isEmpty() )
{
- processDependency( args, trace, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(),
+ processDependency( args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(),
false );
}
+ args.resolver.shutdown();
args.skipper.report();
}
@SuppressWarnings( "checkstyle:parameternumber" )
- private void processDependency( Args args, RequestTrace parent, Results results,
+ private void processDependency( Args args, Results results,
DependencyProcessingContext context, List<Artifact> relocations,
boolean disableVersionManagement )
{
- if ( context.depSelector != null && !context.depSelector.selectDependency( context.dependency ) )
- {
- return;
- }
-
- RequestTrace trace = collectStepTrace( parent, args.request.getRequestContext(), context.parents,
- context.dependency );
- PremanagedDependency preManaged =
- PremanagedDependency.create( context.depManager, context.dependency, disableVersionManagement,
- args.premanagedState );
- Dependency dependency = preManaged.getManagedDependency();
+ Dependency dependency = context.dependency;
+ PremanagedDependency preManaged = context.premanagedDependency;
boolean noDescriptor = isLackingDescriptor( dependency.getArtifact() );
-
boolean traverse =
!noDescriptor && ( context.depTraverser == null || context.depTraverser.traverseDependency(
dependency ) );
- List<? extends Version> versions;
+ Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find( dependency.getArtifact() );
+ DescriptorResolutionResult resolutionResult;
VersionRangeResult rangeResult;
try
{
- VersionRangeRequest rangeRequest = createVersionRangeRequest( args.request.getRequestContext(), trace,
- context.repositories, dependency );
-
- rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session );
-
- versions = filterVersions( dependency, rangeResult, context.verFilter, args.versionContext );
+ resolutionResult = resolutionResultFuture.get();
+ rangeResult = resolutionResult.rangeResult;
}
- catch ( VersionRangeResolutionException e )
+ catch ( Exception e )
{
results.addException( dependency, e, context.parents );
return;
}
- //Resolve newer version first to maximize benefits of skipper
- Collections.reverse( versions );
+ Set<Version> versions = resolutionResult.descriptors.keySet();
for ( Version version : versions )
{
Artifact originalArtifact = dependency.getArtifact().setVersion( version.toString() );
Dependency d = dependency.setArtifact( originalArtifact );
- ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
- args.request.getRequestContext(), trace, context.repositories, d );
-
- final ArtifactDescriptorResult descriptorResult =
- noDescriptor
- ? new ArtifactDescriptorResult( descriptorRequest )
- : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session,
- context.withDependency( d ), results );
-
+ final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get( version );
if ( descriptorResult != null )
{
d = d.setArtifact( descriptorResult.getArtifact() );
@@ -238,8 +246,24 @@ public class BfDependencyCollector
originalArtifact.getGroupId().equals( d.getArtifact().getGroupId() )
&& originalArtifact.getArtifactId().equals( d.getArtifact().getArtifactId() );
- processDependency( args, parent, results, context.withDependency( d ),
- descriptorResult.getRelocations(), disableVersionManagementSubsequently );
+ PremanagedDependency premanagedDependency =
+ PremanagedDependency.create( context.depManager, d, disableVersionManagementSubsequently,
+ args.premanagedState );
+ DependencyProcessingContext relocatedContext =
+ new DependencyProcessingContext( context.depSelector, context.depManager,
+ context.depTraverser, context.verFilter,
+ context.trace, context.repositories, descriptorResult.getManagedDependencies(),
+ context.parents,
+ d, premanagedDependency );
+
+ if ( !filter( relocatedContext ) )
+ {
+ relocatedContext.withDependency( premanagedDependency.getManagedDependency() );
+ resolveArtifactDescriptorAsync( args, relocatedContext, results );
+ processDependency( args, results, relocatedContext, descriptorResult.getRelocations(),
+ disableVersionManagementSubsequently );
+ }
+
return;
}
else
@@ -259,7 +283,8 @@ public class BfDependencyCollector
DependencyProcessingContext parentContext = context.withDependency( d );
if ( recurse )
{
- doRecurse( args, parentContext, descriptorResult, child );
+ doRecurse( args, parentContext, descriptorResult, child, results,
+ disableVersionManagement );
}
else if ( !args.skipper.skipResolution( child, parentContext.parents ) )
{
@@ -284,7 +309,8 @@ public class BfDependencyCollector
@SuppressWarnings( "checkstyle:parameternumber" )
private void doRecurse( Args args, DependencyProcessingContext parentContext,
- ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child )
+ ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child, Results results,
+ boolean disableVersionManagement )
{
DefaultDependencyCollectionContext context = args.collectionContext;
context.set( parentContext.dependency, descriptorResult.getManagedDependencies() );
@@ -319,10 +345,24 @@ public class BfDependencyCollector
parents.add( child );
for ( Dependency dependency : descriptorResult.getDependencies() )
{
- args.dependencyProcessingQueue.add(
+ RequestTrace childTrace =
+ collectStepTrace( parentContext.trace, args.request.getRequestContext(), parents,
+ dependency );
+ PremanagedDependency premanagedDependency =
+ PremanagedDependency.create( childManager, dependency, disableVersionManagement,
+ args.premanagedState );
+ DependencyProcessingContext processingContext =
new DependencyProcessingContext( childSelector, childManager, childTraverser, childFilter,
- childRepos, descriptorResult.getManagedDependencies(), parents, dependency ) );
-
+ childTrace, childRepos, descriptorResult.getManagedDependencies(), parents,
+ dependency, premanagedDependency );
+ if ( !filter( processingContext ) )
+ {
+ //resolve descriptors ahead for managed dependency
+ processingContext.withDependency(
+ processingContext.premanagedDependency.getManagedDependency() );
+ resolveArtifactDescriptorAsync( args, processingContext, results );
+ args.dependencyProcessingQueue.add( processingContext );
+ }
}
args.pool.putChildren( key, child.getChildren() );
args.skipper.cache( child, parents );
@@ -334,6 +374,67 @@ public class BfDependencyCollector
}
}
+ private boolean filter( DependencyProcessingContext context )
+ {
+ return context.depSelector != null && !context.depSelector.selectDependency( context.dependency );
+ }
+
+
+ private void resolveArtifactDescriptorAsync( Args args, DependencyProcessingContext context,
+ Results results )
+ {
+ Dependency dependency = context.dependency;
+ args.resolver.resolveDescriptors( dependency.getArtifact(), () ->
+ {
+ VersionRangeRequest rangeRequest =
+ createVersionRangeRequest( args.request.getRequestContext(), context.trace, context.repositories,
+ dependency );
+ VersionRangeResult rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session );
+ List<? extends Version> versions = filterVersions( dependency, rangeResult, context.verFilter,
+ args.versionContext );
+
+ //resolve newer version first to maximize benefits of skipper
+ Collections.reverse( versions );
+
+ Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>( versions.size() );
+ Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
+ stream.forEach( version ->
+ Optional.ofNullable( resolveDescriptorForVersion( args, context, results, dependency, version ) )
+ .ifPresent( r -> descriptors.put( version, r ) )
+ );
+
+ DescriptorResolutionResult resolutionResult =
+ new DescriptorResolutionResult( dependency.getArtifact(), rangeResult );
+ //keep original sequence
+ versions.forEach( version -> resolutionResult.descriptors.put( version, descriptors.get( version ) ) );
+ //populate for versions in version range
+ resolutionResult.flatten().forEach( dr -> args.resolver.cacheVersionRangeDescriptor( dr.artifact, dr ) );
+
+ return resolutionResult;
+ } );
+ }
+
+ private ArtifactDescriptorResult resolveDescriptorForVersion( Args args, DependencyProcessingContext context,
+ Results results, Dependency dependency,
+ Version version )
+ {
+ Artifact original = dependency.getArtifact();
+ Artifact newArtifact = new DefaultArtifact( original.getGroupId(),
+ original.getArtifactId(), original.getClassifier(), original.getExtension(),
+ version.toString(), original.getProperties(), (ArtifactType) null );
+ Dependency newDependency = new Dependency( newArtifact, dependency.getScope(), dependency.isOptional(),
+ dependency.getExclusions() );
+ DependencyProcessingContext newContext = context.copy();
+
+ ArtifactDescriptorRequest descriptorRequest =
+ createArtifactDescriptorRequest( args.request.getRequestContext(), context.trace,
+ newContext.repositories, newDependency );
+ return isLackingDescriptor( newArtifact )
+ ? new ArtifactDescriptorResult( descriptorRequest )
+ : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session,
+ newContext.withDependency( newDependency ), results );
+ }
+
private ArtifactDescriptorResult resolveCachedArtifactDescriptor( DataPool pool,
ArtifactDescriptorRequest descriptorRequest,
RepositorySystemSession session,
@@ -365,6 +466,89 @@ public class BfDependencyCollector
return descriptorResult;
}
+ static class ParallelDescriptorResolver
+ {
+ final ExecutorService executorService;
+
+ /**
+ * Artifact ID -> Future of DescriptorResolutionResult
+ */
+ final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>( 256 );
+ final Logger logger = LoggerFactory.getLogger( getClass() );
+
+ ParallelDescriptorResolver( RepositorySystemSession session )
+ {
+ this.executorService = getExecutorService( session );
+ }
+
+ void resolveDescriptors( Artifact artifact, Callable<DescriptorResolutionResult> callable )
+ {
+ results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
+ key -> this.executorService.submit( callable ) );
+ }
+
+ void cacheVersionRangeDescriptor( Artifact artifact, DescriptorResolutionResult resolutionResult )
+ {
+ results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
+ key -> ConcurrentUtils.constantFuture( resolutionResult ) );
+ }
+
+ Future<DescriptorResolutionResult> find( Artifact artifact )
+ {
+ return results.get( ArtifactIdUtils.toId( artifact ) );
+ }
+
+ void shutdown()
+ {
+ executorService.shutdown();
+ }
+
+ private ExecutorService getExecutorService( RepositorySystemSession session )
+ {
+ int nThreads = ConfigUtils.getInteger( session, 5, "maven.artifact.threads" );
+ logger.debug( "Created thread pool with {} threads to resolve descriptors.", nThreads );
+ return new ThreadPoolExecutor( nThreads, nThreads, 3L, TimeUnit.SECONDS, new LinkedBlockingQueue(),
+ new WorkerThreadFactory( getClass().getSimpleName() ) );
+ }
+ }
+
+ static class DescriptorResolutionResult
+ {
+ Artifact artifact;
+
+ VersionRangeResult rangeResult;
+
+ Map<Version, ArtifactDescriptorResult> descriptors;
+
+ DescriptorResolutionResult( Artifact artifact, VersionRangeResult rangeResult )
+ {
+ this.artifact = artifact;
+ this.rangeResult = rangeResult;
+ this.descriptors = new LinkedHashMap<>( rangeResult.getVersions().size() );
+ }
+
+ DescriptorResolutionResult( VersionRangeResult rangeResult,
+ Version version, ArtifactDescriptorResult descriptor )
+ {
+ this( descriptor.getArtifact(), rangeResult );
+ this.descriptors.put( version, descriptor );
+ }
+
+ List<DescriptorResolutionResult> flatten()
+ {
+ if ( descriptors.size() > 1 )
+ {
+ return descriptors.entrySet().stream()
+ .map( e -> new DescriptorResolutionResult( rangeResult, e.getKey(), e.getValue() ) )
+ .collect( Collectors.toList() );
+ }
+ else
+ {
+ return Collections.emptyList();
+ }
+ }
+ }
+
static class Args
{
@@ -386,9 +570,12 @@ public class BfDependencyCollector
final DependencyResolutionSkipper skipper;
+ final ParallelDescriptorResolver resolver;
+
Args( RepositorySystemSession session, DataPool pool,
- DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext,
- CollectRequest request, DependencyResolutionSkipper skipper )
+ DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext,
+ CollectRequest request, DependencyResolutionSkipper skipper,
+ ParallelDescriptorResolver resolver )
{
this.session = session;
this.request = request;
@@ -398,6 +585,7 @@ public class BfDependencyCollector
this.collectionContext = collectionContext;
this.versionContext = versionContext;
this.skipper = skipper;
+ this.resolver = resolver;
}
}
diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/DependencyProcessingContext.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/DependencyProcessingContext.java
index ee2e7720..508d2d04 100644
--- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/DependencyProcessingContext.java
+++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/DependencyProcessingContext.java
@@ -21,12 +21,14 @@ package org.eclipse.aether.internal.impl.collect.bf;
import java.util.List;
+import org.eclipse.aether.RequestTrace;
import org.eclipse.aether.collection.DependencyManager;
import org.eclipse.aether.collection.DependencySelector;
import org.eclipse.aether.collection.DependencyTraverser;
import org.eclipse.aether.collection.VersionFilter;
import org.eclipse.aether.graph.Dependency;
import org.eclipse.aether.graph.DependencyNode;
+import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
import org.eclipse.aether.repository.RemoteRepository;
/**
@@ -47,6 +49,8 @@ final class DependencyProcessingContext
* All parents of the dependency in the top > down order.
*/
final List<DependencyNode> parents;
+ final PremanagedDependency premanagedDependency;
+ final RequestTrace trace;
Dependency dependency;
@SuppressWarnings( "checkstyle:parameternumber" )
@@ -54,17 +58,21 @@ final class DependencyProcessingContext
DependencyManager depManager,
DependencyTraverser depTraverser,
VersionFilter verFilter,
+ RequestTrace trace,
List<RemoteRepository> repositories,
List<Dependency> managedDependencies,
List<DependencyNode> parents,
- Dependency dependency )
+ Dependency dependency,
+ PremanagedDependency premanagedDependency )
{
this.depSelector = depSelector;
this.depManager = depManager;
this.depTraverser = depTraverser;
this.verFilter = verFilter;
+ this.trace = trace;
this.repositories = repositories;
this.dependency = dependency;
+ this.premanagedDependency = premanagedDependency;
this.managedDependencies = managedDependencies;
this.parents = parents;
}
@@ -75,6 +83,13 @@ final class DependencyProcessingContext
return this;
}
+ DependencyProcessingContext copy()
+ {
+ return new DependencyProcessingContext( depSelector, depManager, depTraverser,
+ verFilter, trace, repositories, managedDependencies, parents, dependency,
+ premanagedDependency );
+ }
+
DependencyNode getParent()
{
return parents.get( parents.size() - 1 );