You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2020/08/10 11:53:45 UTC
[maven-resolver] 02/02: Try
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch redisson
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
commit ca58718e414de817c17c1e1ee5294d813e1881c3
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Sun Aug 9 20:05:53 2020 +0200
Try
---
.../synccontext/GlobalSyncContextFactory.java | 86 -------
.../pom.xml | 18 +-
.../internal/impl/DefaultSyncContextFactory.java | 4 +-
.../aether/internal/impl/TrackingFileManager.java | 0
.../synccontext/RedissonSyncContextFactory.java | 267 +++++++++++++++++++++
pom.xml | 2 +-
6 files changed, 284 insertions(+), 93 deletions(-)
diff --git a/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/synccontext/GlobalSyncContextFactory.java b/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/synccontext/GlobalSyncContextFactory.java
deleted file mode 100644
index 83d91ba..0000000
--- a/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/synccontext/GlobalSyncContextFactory.java
+++ /dev/null
@@ -1,86 +0,0 @@
-package org.eclipse.aether.synccontext;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.util.Collection;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.eclipse.aether.RepositorySystemSession;
-import org.eclipse.aether.SyncContext;
-import org.eclipse.aether.artifact.Artifact;
-import org.eclipse.aether.impl.SyncContextFactory;
-import org.eclipse.aether.metadata.Metadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A factory to create synchronization contexts. This implementation uses fair global locking
- * based on {@link ReentrantReadWriteLock}. Explicit artifacts and metadata passed are ignored.
- * <br>
- * This factory is considered experimental.
- */
-public class GlobalSyncContextFactory
- implements SyncContextFactory
-{
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock( true );
-
- public SyncContext newInstance( RepositorySystemSession session, boolean shared )
- {
- return new GlobalSyncContext( shared ? lock.readLock() : lock.writeLock(), shared );
- }
-
- static class GlobalSyncContext
- implements SyncContext
- {
- private static final Logger LOGGER = LoggerFactory.getLogger( GlobalSyncContext.class );
-
- private final Lock lock;
- private final boolean shared;
- private int lockHoldCount;
-
- GlobalSyncContext( Lock lock, boolean shared )
- {
- this.lock = lock;
- this.shared = shared;
- }
-
- public void acquire( Collection<? extends Artifact> artifact, Collection<? extends Metadata> metadata )
- {
- LOGGER.trace( "Acquiring global {} lock (currently held: {})",
- shared ? "read" : "write", lockHoldCount );
- lock.lock();
- lockHoldCount++;
- }
-
- public void close()
- {
- while ( lockHoldCount != 0 )
- {
- LOGGER.trace( "Releasing global {} lock (currently held: {})",
- shared ? "read" : "write", lockHoldCount );
- lock.unlock();
- lockHoldCount--;
- }
- }
-
- }
-
-}
diff --git a/maven-resolver-synccontext-global/pom.xml b/maven-resolver-synccontext-redisson/pom.xml
similarity index 83%
rename from maven-resolver-synccontext-global/pom.xml
rename to maven-resolver-synccontext-redisson/pom.xml
index f70fee1..333b6ce 100644
--- a/maven-resolver-synccontext-global/pom.xml
+++ b/maven-resolver-synccontext-redisson/pom.xml
@@ -28,16 +28,17 @@
<version>1.5.1-SNAPSHOT</version>
</parent>
- <artifactId>maven-resolver-synccontext-global</artifactId>
+ <artifactId>maven-resolver-synccontext-redisson</artifactId>
- <name>Maven Artifact Resolver Sync Context Global</name>
+ <name>Maven Artifact Resolver Sync Context Redisson</name>
<description>
- A sync context implementation using a global fair lock.
+ A sync context implementation using Redisson distributed locks.
</description>
<properties>
- <Automatic-Module-Name>org.apache.maven.resolver.synccontext.global</Automatic-Module-Name>
+ <Automatic-Module-Name>org.apache.maven.resolver.synccontext.redisson</Automatic-Module-Name>
<Bundle-SymbolicName>${Automatic-Module-Name}</Bundle-SymbolicName>
+ <javaVersion>8</javaVersion>
</properties>
<dependencies>
@@ -50,6 +51,10 @@
<artifactId>maven-resolver-impl</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-util</artifactId>
+ </dependency>
+ <dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
<scope>provided</scope>
@@ -59,6 +64,11 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.redisson</groupId>
+ <artifactId>redisson</artifactId>
+ <version>3.13.3</version>
+ </dependency>
</dependencies>
<build>
diff --git a/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java b/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java
similarity index 91%
rename from maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java
rename to maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java
index 5386359..17961cf 100644
--- a/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java
+++ b/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/internal/impl/DefaultSyncContextFactory.java
@@ -22,7 +22,7 @@ package org.eclipse.aether.internal.impl;
import javax.inject.Named;
import javax.inject.Singleton;
-import org.eclipse.aether.synccontext.GlobalSyncContextFactory;
+import org.eclipse.aether.synccontext.RedissonSyncContextFactory;
/**
* This is a shim to override (shadow) the actual {@code DefaultSyncContextFactory}} via ext classpath.
@@ -30,7 +30,7 @@ import org.eclipse.aether.synccontext.GlobalSyncContextFactory;
@Named
@Singleton
public class DefaultSyncContextFactory
- extends GlobalSyncContextFactory
+ extends RedissonSyncContextFactory
{
}
diff --git a/maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/internal/impl/TrackingFileManager.java b/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/internal/impl/TrackingFileManager.java
similarity index 100%
rename from maven-resolver-synccontext-global/src/main/java/org/eclipse/aether/internal/impl/TrackingFileManager.java
rename to maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/internal/impl/TrackingFileManager.java
diff --git a/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/synccontext/RedissonSyncContextFactory.java b/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/synccontext/RedissonSyncContextFactory.java
new file mode 100644
index 0000000..cc0886b
--- /dev/null
+++ b/maven-resolver-synccontext-redisson/src/main/java/org/eclipse/aether/synccontext/RedissonSyncContextFactory.java
@@ -0,0 +1,267 @@
+package org.eclipse.aether.synccontext;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.TreeSet;
+
+import org.eclipse.aether.RepositorySystemSession;
+import org.eclipse.aether.SyncContext;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.impl.SyncContextFactory;
+import org.eclipse.aether.metadata.Metadata;
+import org.eclipse.aether.util.ChecksumUtils;
+import org.eclipse.aether.util.ConfigUtils;
+import org.redisson.Redisson;
+import org.redisson.api.RLock;
+import org.redisson.api.RReadWriteLock;
+import org.redisson.api.RedissonClient;
+import org.redisson.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A factory to create synchronization contexts. This implementation uses Redis, more specifically
+ * its {@link RReadWriteLock}. It locks -- very fine-grained -- down to an artifact/metadata version
+ * if required. <br>
+ * This factory is considered experimental.
+ */
+public class RedissonSyncContextFactory implements SyncContextFactory {
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "maven-resolver-redisson.yaml";
+ private static final String DEFAULT_REDIS_ADDRESS = "redis://localhost:6379";
+ private static final String DEFAULT_CLIENT_NAME = "maven-resolver";
+ private static final String DEFAULT_DISCRIMINATOR = "default";
+ private static final String DEFAULT_DISCRIMINATOR_DIGEST = "2923f6fa36614586ea09b4424b438915cc1b9b67";
+
+ private static final String CONFIG_PROP_CONFIG_FILE = "aether.syncContext.redisson.configFile";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RedissonSyncContextFactory.class);
+
+ private final RedissonClient redissonClient;
+ private final String localhostDiscriminator;
+
+ public RedissonSyncContextFactory() {
+ this.redissonClient = createRedissonClient();
+ this.localhostDiscriminator = createLocalhostDiscriminator();
+ }
+
+ private RedissonClient createRedissonClient() {
+ Path configFilePath = null;
+
+ String configFile = ConfigUtils.getString(System.getProperties(), null, CONFIG_PROP_CONFIG_FILE);
+ if (configFile != null && !configFile.isEmpty()) {
+ configFilePath = Paths.get(configFile);
+ if (Files.notExists(configFilePath)) {
+ throw new IllegalArgumentException("The specified Redisson config file does not exist: " + configFilePath);
+ }
+ }
+
+ if (configFilePath == null) {
+ String mavenConf = ConfigUtils.getString(System.getProperties(), null, "maven.conf");
+ if (mavenConf != null && !mavenConf.isEmpty()) {
+ configFilePath = Paths.get(mavenConf, DEFAULT_CONFIG_FILE_NAME);
+ if (Files.notExists(configFilePath)) {
+ configFilePath = null;
+ }
+ }
+ }
+
+ Config config = null;
+
+ if (configFilePath != null) {
+ LOGGER.trace("Reading Redisson config file from '{}'", configFilePath);
+ try (InputStream is = Files.newInputStream(configFilePath)) {
+ config = Config.fromYAML(is);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to read Redisson config file: " + configFilePath, e);
+ }
+ } else {
+ config = new Config();
+ config.useSingleServer().setAddress(DEFAULT_REDIS_ADDRESS)
+ .setClientName(DEFAULT_CLIENT_NAME);
+ }
+
+ RedissonClient redissonClient = Redisson.create(config);
+ LOGGER.trace("Created Redisson client with id '{}'", redissonClient.getId());
+
+ return redissonClient;
+ }
+
+ private String createLocalhostDiscriminator() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ LOGGER.trace("Failed to calculate localhost descriminator, using '{}'",
+ DEFAULT_DISCRIMINATOR, e);
+ return DEFAULT_DISCRIMINATOR;
+ }
+ }
+
+ public SyncContext newInstance(RepositorySystemSession session, boolean shared) {
+ return new RedissonSyncContext(session, localhostDiscriminator, redissonClient, shared);
+ }
+
+ static class RedissonSyncContext implements SyncContext {
+
+ private static final String CONFIG_DISCRIMINATOR = "aether.syncContext.redisson.discriminator";
+
+ private static final String KEY_PREFIX = "maven:resolver:";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RedissonSyncContext.class);
+
+ private final RepositorySystemSession session;
+ private final String localhostDiscriminator;
+ private final RedissonClient redissonClient;
+ private final boolean shared;
+ private final Map<String, RReadWriteLock> locks = new LinkedHashMap<>();
+
+ private RedissonSyncContext(RepositorySystemSession session, String localhostDiscriminator,
+ RedissonClient redisson, boolean shared) {
+ this.session = session;
+ this.localhostDiscriminator = localhostDiscriminator;
+ this.redissonClient = redisson;
+ this.shared = shared;
+ }
+
+ public void acquire(Collection<? extends Artifact> artifacts,
+ Collection<? extends Metadata> metadatas) {
+ String discriminator = createDiscriminator();
+ LOGGER.trace("Using Redis key discriminator '{}' during this session", discriminator);
+
+ // Deadlock prevention: https://stackoverflow.com/a/16780988/696632
+ // We must acquire multiple locks always in the same order!
+ Collection<String> keys = new TreeSet<>();
+ if (artifacts != null) {
+ for (Artifact artifact : artifacts) {
+ // TODO Should we include extension and classifier too?
+ String key = "artifact:" + artifact.getGroupId() + ":"
+ + artifact.getArtifactId() + ":" + artifact.getBaseVersion();
+ keys.add(key);
+ }
+ }
+
+ if (metadatas != null) {
+ for (Metadata metadata : metadatas) {
+ String key;
+ if (!metadata.getGroupId().isEmpty()) {
+ StringBuilder keyBuilder = new StringBuilder("metadata:");
+ keyBuilder.append(metadata.getGroupId());
+ if (!metadata.getArtifactId().isEmpty()) {
+ keyBuilder.append(':').append(metadata.getArtifactId());
+ if (!metadata.getVersion().isEmpty()) {
+ keyBuilder.append(':').append(metadata.getVersion());
+ }
+ }
+ key = keyBuilder.toString();
+ } else {
+ key = "metadata:ROOT";
+ }
+ keys.add(key);
+ }
+ }
+
+ LOGGER.trace("Need {} {} lock(s) for {}", keys.size(), shared ? "read" : "write", keys);
+
+ for (String key : keys) {
+ RReadWriteLock rwLock = locks.get(key);
+ if (rwLock == null) {
+ rwLock = redissonClient
+ .getReadWriteLock(KEY_PREFIX + discriminator + ":" + key);
+ locks.put(key, rwLock);
+ }
+
+ RLock actualLock = shared ? rwLock.readLock() : rwLock.writeLock();
+ LOGGER.trace("Acquiring {} lock for '{}' (currently held: {}, already locked: {})",
+ shared ? "read" : "write", key, actualLock.getHoldCount(),
+ actualLock.isLocked());
+ // If this still produces a deadlock we might need to switch to #tryLock() with n
+ // attempts
+ actualLock.lock();
+ }
+ // TODO This is probably wrong. In a subsequent (reentrant) call we already have acquired
+ // locks previously. We need to could locks for a single acquire call only
+ LOGGER.trace("Total locks acquired: {}", locks.size());
+ }
+
+ private String createDiscriminator() {
+ String discriminator = ConfigUtils.getString(session, null, CONFIG_DISCRIMINATOR);
+
+ if (discriminator == null || discriminator.isEmpty()) {
+
+ File basedir = session.getLocalRepository().getBasedir();
+ discriminator = localhostDiscriminator + ":" + basedir;
+ try {
+ Map<String, Object> checksums = ChecksumUtils.calc(
+ discriminator.toString().getBytes(StandardCharsets.UTF_8),
+ Collections.singletonList("SHA-1"));
+ Object checksum = checksums.get("SHA-1");
+
+ if (checksum instanceof Exception)
+ throw (Exception) checksum;
+
+ return String.valueOf(checksum);
+ } catch (Exception e) {
+ LOGGER.trace("Failed to calculate discriminator digest, using '{}'",
+ DEFAULT_DISCRIMINATOR_DIGEST, e);
+ return DEFAULT_DISCRIMINATOR_DIGEST;
+ }
+ }
+
+ return discriminator;
+ }
+
+ public void close() {
+ // Release locks in reverse insertion order
+ Deque<String> keys = new LinkedList<>(locks.keySet());
+ Iterator<String> keysIter = keys.descendingIterator();
+ while (keysIter.hasNext()) {
+ String key = keysIter.next();
+ RReadWriteLock rwLock = locks.get(key);
+ RLock actualLock = shared ? rwLock.readLock() : rwLock.writeLock();
+ while (actualLock.getHoldCount() > 0) {
+ LOGGER.trace("Releasing {} lock for '{}' (currently held: {})",
+ shared ? "read" : "write", key, actualLock.getHoldCount());
+ actualLock.unlock();
+ }
+ }
+ // TODO Should we count reentrant ones too?
+ LOGGER.trace("Total locks released: {}", locks.size());
+ locks.clear();
+ }
+
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 2f7c4b2..30aa37b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,7 +90,7 @@
<module>maven-resolver-transport-http</module>
<module>maven-resolver-transport-wagon</module>
<module>maven-resolver-demos</module>
- <module>maven-resolver-synccontext-global</module>
+ <module>maven-resolver-synccontext-redisson</module>
</modules>
<dependencyManagement>