You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2023/02/15 20:51:10 UTC
[maven-resolver] branch master updated: [MRESOLVER-319] Parallel deploy, pt2 (#245)
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
The following commit(s) were added to refs/heads/master by this push:
new 26f00f3e [MRESOLVER-319] Parallel deploy, pt2 (#245)
26f00f3e is described below
commit 26f00f3edc9d391c42e32f7cf5e0b1a1326271f4
Author: Tamas Cservenak <ta...@cservenak.net>
AuthorDate: Wed Feb 15 21:51:04 2023 +0100
[MRESOLVER-319] Parallel deploy, pt2 (#245)
In the mean time turns out we have two important "implementation details" to obey, and behave correctly:
We need ONE transport to happen (go thru pipe fully) before we can pour the rest onto transport
The HTTP transporter is equipped with "shared" caches for auth that is heavily used by HttpClient to make decisions (ie. about re-attempting request). Current code is quite complicated, but auth sharing happens when one transfer task is done, and that step "inseminates" the cache to be used by others (and skip the auth dance from that point onwards). Similar step happens by transfer by issuing OPTIONS to remote to discover does it deals with WebDAV server or not (is MKCOL dance needed [...]
We must ensure proper ordering, that was not ensured by origina implementation
Clients when going for metadata are getting them as G -> A -> V (the longest chain in case of snapshot maven plugin). Hence, we must ensure we deploy the opposite order, and we start deploying next group ONLY when we are fully done with previous group. Not ensuring this would leave a window of opportunity of failed build, where for example A level md is present, but V level not yet.
---
https://issues.apache.org/jira/browse/MRESOLVER-319
---
.../connector/basic/BasicRepositoryConnector.java | 119 ++++++++++++++++++---
1 file changed, 105 insertions(+), 14 deletions(-)
diff --git a/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java b/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
index e7ff2e94..60e83b99 100644
--- a/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
+++ b/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.aether.ConfigurationProperties;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.RequestTrace;
+import org.eclipse.aether.metadata.Metadata;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.spi.connector.ArtifactDownload;
import org.eclipse.aether.spi.connector.ArtifactUpload;
@@ -217,6 +218,8 @@ final class BasicRepositoryConnector
RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
List<ChecksumAlgorithmFactory> checksumAlgorithmFactories = layout.getChecksumAlgorithmFactories();
+ boolean first = true;
+
for ( MetadataDownload transfer : safeMetadataDownloads )
{
URI location = layout.getLocation( transfer.getMetadata(), false );
@@ -234,7 +237,15 @@ final class BasicRepositoryConnector
Runnable task = new GetTaskRunner( location, transfer.getFile(), checksumPolicy,
checksumAlgorithmFactories, checksumLocations, null, listener );
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
for ( ArtifactDownload transfer : safeArtifactDownloads )
@@ -275,7 +286,15 @@ final class BasicRepositoryConnector
task = new GetTaskRunner( location, transfer.getFile(), checksumPolicy,
checksumAlgorithmFactories, checksumLocations, providedChecksums, listener );
}
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
errorForwarder.await();
@@ -293,6 +312,8 @@ final class BasicRepositoryConnector
Executor executor = getExecutor( parallelPut ? safeArtifactUploads.size() + safeMetadataUploads.size() : 1 );
RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
+ boolean first = true;
+
for ( ArtifactUpload transfer : safeArtifactUploads )
{
URI location = layout.getLocation( transfer.getArtifact(), true );
@@ -306,29 +327,99 @@ final class BasicRepositoryConnector
Runnable task = new PutTaskRunner( location, transfer.getFile(), transfer.getFileTransformer(),
checksumLocations, listener );
-
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
errorForwarder.await(); // make sure all artifacts are PUT before we go with Metadata
- for ( MetadataUpload transfer : safeMetadataUploads )
+ for ( List<? extends MetadataUpload> transferGroup : groupUploads( safeMetadataUploads ) )
{
- URI location = layout.getLocation( transfer.getMetadata(), true );
+ for ( MetadataUpload transfer : transferGroup )
+ {
+ URI location = layout.getLocation( transfer.getMetadata(), true );
- TransferResource resource = newTransferResource( location, transfer.getFile(), transfer.getTrace() );
- TransferEvent.Builder builder = newEventBuilder( resource, true, false );
- MetadataTransportListener listener = new MetadataTransportListener( transfer, repository, builder );
+ TransferResource resource = newTransferResource( location, transfer.getFile(), transfer.getTrace() );
+ TransferEvent.Builder builder = newEventBuilder( resource, true, false );
+ MetadataTransportListener listener = new MetadataTransportListener( transfer, repository, builder );
- List<RepositoryLayout.ChecksumLocation> checksumLocations =
- layout.getChecksumLocations( transfer.getMetadata(), true, location );
+ List<RepositoryLayout.ChecksumLocation> checksumLocations =
+ layout.getChecksumLocations( transfer.getMetadata(), true, location );
- Runnable task = new PutTaskRunner( location, transfer.getFile(), checksumLocations, listener );
+ Runnable task = new PutTaskRunner( location, transfer.getFile(), checksumLocations, listener );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
+ }
- executor.execute( errorForwarder.wrap( task ) );
+ errorForwarder.await(); // make sure each group is done before starting next group
}
+ }
- errorForwarder.await();
+ /**
+ * This method "groups" the Metadata to be uploaded by their level (version, artifact, group and root). This is MUST
+ * as clients consume metadata in opposite order (root, group, artifact, version), and hence, we must deploy and
+ * ensure (in case of parallel deploy) that all V level metadata is deployed before we start deploying A level, etc.
+ */
+ private static List<List<MetadataUpload>> groupUploads( Collection<? extends MetadataUpload> metadataUploads )
+ {
+ ArrayList<MetadataUpload> v = new ArrayList<>();
+ ArrayList<MetadataUpload> a = new ArrayList<>();
+ ArrayList<MetadataUpload> g = new ArrayList<>();
+ ArrayList<MetadataUpload> r = new ArrayList<>();
+
+ for ( MetadataUpload transfer : metadataUploads )
+ {
+ Metadata metadata = transfer.getMetadata();
+ if ( !"".equals( metadata.getVersion() ) )
+ {
+ v.add( transfer );
+ }
+ else if ( !"".equals( metadata.getArtifactId() ) )
+ {
+ a.add( transfer );
+ }
+ else if ( !"".equals( metadata.getGroupId() ) )
+ {
+ g.add( transfer );
+ }
+ else
+ {
+ r.add( transfer );
+ }
+ }
+
+ List<List<MetadataUpload>> result = new ArrayList<>( 4 );
+ if ( !v.isEmpty() )
+ {
+ result.add( v );
+ }
+ if ( !a.isEmpty() )
+ {
+ result.add( a );
+ }
+ if ( !g.isEmpty() )
+ {
+ result.add( g );
+ }
+ if ( !r.isEmpty() )
+ {
+ result.add( r );
+ }
+ return result;
}
private static <T> Collection<T> safe( Collection<T> items )