You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@karaf.apache.org by jb...@apache.org on 2018/02/09 06:57:01 UTC
[karaf] branch karaf-4.1.x updated: [KARAF-5604] Speed up
features-generate-descriptor
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch karaf-4.1.x
in repository https://gitbox.apache.org/repos/asf/karaf.git
The following commit(s) were added to refs/heads/karaf-4.1.x by this push:
new b9a8997 [KARAF-5604] Speed up features-generate-descriptor
b9a8997 is described below
commit b9a899749163529ac188f0562693d231991c30a7
Author: Robert Varga <ni...@hq.sk>
AuthorDate: Thu Feb 8 15:42:22 2018 +0100
[KARAF-5604] Speed up features-generate-descriptor
OpenDaylight uses features-generate-descriptor to process a rather
large set (~196) of features being used as dependencies of a generated
feature. These features also contain a large number of bundles.
This patch introduces an explicit SimplLRUCache based on LinkedHashMap,
and places it into both GenerateDescriptorMojo and Dependency31Helper.
The size of the two cache instances can be controlled via plugin configuration
and default to 256 and 1024 entries respectively.
Signed-off-by: Robert Varga <ni...@hq.sk>
---
.../tooling/features/GenerateDescriptorMojo.java | 57 +++++++++++++++-------
.../karaf/tooling/utils/Dependency31Helper.java | 53 +++++++++++++-------
.../tooling/utils/DependencyHelperFactory.java | 6 +--
.../apache/karaf/tooling/utils/SimpleLRUCache.java | 45 +++++++++++++++++
4 files changed, 122 insertions(+), 39 deletions(-)
diff --git a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/features/GenerateDescriptorMojo.java b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/features/GenerateDescriptorMojo.java
index a36f191..0d0301f 100644
--- a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/features/GenerateDescriptorMojo.java
+++ b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/features/GenerateDescriptorMojo.java
@@ -39,7 +39,6 @@ import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.WeakHashMap;
import java.util.jar.JarInputStream;
import java.util.jar.Manifest;
@@ -60,6 +59,7 @@ import org.apache.karaf.tooling.utils.LocalDependency;
import org.apache.karaf.tooling.utils.ManifestUtils;
import org.apache.karaf.tooling.utils.MavenUtil;
import org.apache.karaf.tooling.utils.MojoSupport;
+import org.apache.karaf.tooling.utils.SimpleLRUCache;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.resolver.ArtifactNotFoundException;
import org.apache.maven.artifact.resolver.ArtifactResolutionException;
@@ -91,7 +91,7 @@ import org.xml.sax.SAXException;
/**
* Generates the features XML file starting with an optional source feature.xml and adding
* project dependencies as bundles and feature/car dependencies.
- *
+ *
* NB this requires a recent maven-install-plugin such as 2.3.1
*/
@Mojo(name = "features-generate-descriptor", defaultPhase = LifecyclePhase.COMPILE, requiresDependencyResolution = ResolutionScope.RUNTIME, threadSafe = true)
@@ -235,14 +235,14 @@ public class GenerateDescriptorMojo extends MojoSupport {
*/
@Parameter(defaultValue = "${project.artifactId}")
private String primaryFeatureName;
-
+
/**
* Flag indicating whether bundles should use the version range declared in the POM. If <code>false</code>,
* the actual version of the resolved artifacts will be used.
*/
@Parameter(defaultValue = "false")
private boolean useVersionRange;
-
+
/**
* Flag indicating whether the plugin should determine whether transitive dependencies are declared with
* a version range. If this flag is set to <code>true</code> and a transitive dependency has been found
@@ -265,6 +265,19 @@ public class GenerateDescriptorMojo extends MojoSupport {
private boolean simplifyBundleDependencies;
/**
+ * Maximum size of the artifact LRU cache. This cache is used to prevent repeated artifact-to-file resolution.
+ */
+ @Parameter(defaultValue = "1024")
+ private int artifactCacheSize;
+
+ /**
+ * Maximum size of the Features LRU cache. This cache is used to prevent repeated deserialization of features
+ * XML files.
+ */
+ @Parameter(defaultValue = "256")
+ private int featuresCacheSize;
+
+ /**
* Name of features which are prerequisites (they still need to be defined separately).
*/
@Parameter
@@ -286,7 +299,7 @@ public class GenerateDescriptorMojo extends MojoSupport {
*/
@Component
private PlexusContainer container;
-
+
@Component
private RepositorySystem repoSystem;
@@ -295,7 +308,7 @@ public class GenerateDescriptorMojo extends MojoSupport {
@Component
protected MavenFileFilter mavenFileFilter;
-
+
@Component
private ProjectBuilder mavenProjectBuilder;
@@ -310,11 +323,12 @@ public class GenerateDescriptorMojo extends MojoSupport {
// maven log
private Log log;
-
+
// If useVersionRange is true, this map will be used to cache
// resolved MavenProjects
private final Map<Artifact, MavenProject> resolvedProjects = new HashMap<>();
+ @Override
public void execute() throws MojoExecutionException, MojoFailureException {
try {
if (enableGeneration == null) {
@@ -334,7 +348,8 @@ public class GenerateDescriptorMojo extends MojoSupport {
}
}
- this.dependencyHelper = DependencyHelperFactory.createDependencyHelper(this.container, this.project, this.mavenSession, getLog());
+ this.dependencyHelper = DependencyHelperFactory.createDependencyHelper(this.container, this.project,
+ this.mavenSession, this.artifactCacheSize, getLog());
this.dependencyHelper.getDependencies(project, includeTransitiveDependency);
this.localDependencies = dependencyHelper.getLocalDependencies();
this.treeListing = dependencyHelper.getTreeListing();
@@ -363,11 +378,11 @@ public class GenerateDescriptorMojo extends MojoSupport {
resolvedProject = resolvedProjects.get(artifact);
if (resolvedProject == null) {
final ProjectBuildingRequest request = new DefaultProjectBuildingRequest();
-
- // Fixes KARAF-4626; if the system properties are not transferred to the request,
+
+ // Fixes KARAF-4626; if the system properties are not transferred to the request,
// test-feature-use-version-range-transfer-properties will fail
request.setSystemProperties(System.getProperties());
-
+
request.setResolveDependencies(true);
request.setRemoteRepositories(project.getPluginArtifactRepositories());
request.setLocalRepository(localRepo);
@@ -402,7 +417,7 @@ public class GenerateDescriptorMojo extends MojoSupport {
}
return versionOrRange;
}
-
+
/*
* Write all project dependencies as feature
*/
@@ -460,7 +475,7 @@ public class GenerateDescriptorMojo extends MojoSupport {
// TODO Initialise the repositories from the existing feature file if any
Map<Dependency, Feature> otherFeatures = new HashMap<>();
Map<Feature, String> featureRepositories = new HashMap<>();
- FeaturesCache cache = new FeaturesCache();
+ FeaturesCache cache = new FeaturesCache(featuresCacheSize);
for (final LocalDependency entry : localDependencies) {
Object artifact = entry.getArtifact();
@@ -536,7 +551,7 @@ public class GenerateDescriptorMojo extends MojoSupport {
wrapDependency.setPrerequisite(true);
feature.getFeature().add(wrapDependency);
}
-
+
if ((!feature.getBundle().isEmpty() || !feature.getFeature().isEmpty()) && !features.getFeature().contains(feature)) {
features.getFeature().add(feature);
}
@@ -672,14 +687,16 @@ public class GenerateDescriptorMojo extends MojoSupport {
}
}
- private static Features readFeaturesFile(File featuresFile) throws XMLStreamException, JAXBException, IOException {
+ static Features readFeaturesFile(File featuresFile) throws XMLStreamException, JAXBException, IOException {
return JaxbUtil.unmarshal(featuresFile.toURI().toASCIIString(), false);
}
+ @Override
public void setLog(Log log) {
this.log = log;
}
+ @Override
public Log getLog() {
if (log == null) {
setLog(new SystemStreamLog());
@@ -946,16 +963,20 @@ public class GenerateDescriptorMojo extends MojoSupport {
}
private static final class FeaturesCache {
- private final Map<File, Features> map = new WeakHashMap<>();
+ private final SimpleLRUCache<File, Features> cache;
+
+ FeaturesCache(int featuresCacheSize) {
+ cache = new SimpleLRUCache<>(featuresCacheSize);
+ }
Features get(final File featuresFile) throws XMLStreamException, JAXBException, IOException {
- final Features existing = map.get(featuresFile);
+ final Features existing = cache.get(featuresFile);
if (existing != null) {
return existing;
}
final Features computed = readFeaturesFile(featuresFile);
- map.put(featuresFile, computed);
+ cache.put(featuresFile, computed);
return computed;
}
}
diff --git a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/Dependency31Helper.java b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/Dependency31Helper.java
index 89a1762..823b7ad 100644
--- a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/Dependency31Helper.java
+++ b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/Dependency31Helper.java
@@ -74,25 +74,32 @@ public class Dependency31Helper implements DependencyHelper {
*/
private final List<RemoteRepository> projectRepositories;
+ private final SimpleLRUCache<Artifact, ArtifactResult> artifactCache;
+
// dependencies we are interested in
protected Set<LocalDependency> localDependencies;
// log of what happened during search
protected String treeListing;
@SuppressWarnings("unchecked")
- public Dependency31Helper(List<?> repositories, Object session, RepositorySystem repositorySystem) {
+ public Dependency31Helper(List<?> repositories, Object session, RepositorySystem repositorySystem, int cacheSize) {
this.projectRepositories = (List<RemoteRepository>) repositories;
this.repositorySystemSession = (RepositorySystemSession) session;
this.repositorySystem = repositorySystem;
+ this.artifactCache = new SimpleLRUCache<>(cacheSize);
+ }
+
+ public Dependency31Helper(List<?> repositories, Object session, RepositorySystem repositorySystem) {
+ this(repositories, session, repositorySystem, 32);
+ }
+
+ public void setRepositorySession(final ProjectBuildingRequest request) throws MojoExecutionException {
+ try {
+ invokeMethod(request, "setRepositorySession", repositorySystemSession);
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+ throw new MojoExecutionException("Cannot set repository session on project building request", e);
+ }
}
-
- public void setRepositorySession(final ProjectBuildingRequest request) throws MojoExecutionException {
- try {
- invokeMethod(request, "setRepositorySession", repositorySystemSession);
- } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
- throw new MojoExecutionException("Cannot set repository session on project building request", e);
- }
- }
@Override
public Set<LocalDependency> getLocalDependencies() {
@@ -283,7 +290,7 @@ public class Dependency31Helper implements DependencyHelper {
public boolean isArtifactAFeature(Object artifact) {
return Dependency31Helper.isFeature((Artifact) artifact);
}
-
+
@Override
public String getBaseVersion(Object artifact) {
return ((Artifact) artifact).getBaseVersion();
@@ -304,17 +311,30 @@ public class Dependency31Helper implements DependencyHelper {
return ((Artifact) artifact).getClassifier();
}
- @Override
- public File resolve(Object artifact, Log log) {
+ private ArtifactResult resolveArtifact(Artifact artifact) throws ArtifactResolutionException {
+ ArtifactResult result = artifactCache.get(artifact);
+ if (result != null) {
+ return result;
+ }
+
ArtifactRequest request = new ArtifactRequest();
- request.setArtifact((Artifact) artifact);
+ request.setArtifact(artifact);
request.setRepositories(projectRepositories);
+ result = repositorySystem.resolveArtifact(repositorySystemSession, request);
+ if (result != null) {
+ artifactCache.put(artifact, result);
+ }
+ return result;
+ }
+
+ @Override
+ public File resolve(Object artifact, Log log) {
log.debug("Resolving artifact " + artifact + " from " + projectRepositories);
ArtifactResult result;
try {
- result = repositorySystem.resolveArtifact(repositorySystemSession, request);
+ result = resolveArtifact((Artifact) artifact);
} catch (ArtifactResolutionException e) {
log.warn("Cound not resolve " + artifact, e);
return null;
@@ -336,15 +356,12 @@ public class Dependency31Helper implements DependencyHelper {
}
}
id = MavenUtil.mvnToAether(id);
- ArtifactRequest request = new ArtifactRequest();
- request.setArtifact(new DefaultArtifact(id));
- request.setRepositories(projectRepositories);
log.debug("Resolving artifact " + id + " from " + projectRepositories);
ArtifactResult result;
try {
- result = repositorySystem.resolveArtifact(repositorySystemSession, request);
+ result = resolveArtifact(new DefaultArtifact(id));
} catch (ArtifactResolutionException e) {
log.warn("Could not resolve " + id, e);
throw new MojoFailureException(format("Couldn't resolve artifact %s", id), e);
diff --git a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/DependencyHelperFactory.java b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/DependencyHelperFactory.java
index e60edf9..5ca3311 100644
--- a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/DependencyHelperFactory.java
+++ b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/DependencyHelperFactory.java
@@ -49,7 +49,7 @@ public class DependencyHelperFactory {
* @return The {@link DependencyHelper} depending of the Maven version used.
* @throws MojoExecutionException If the plugin execution fails.
*/
- public static DependencyHelper createDependencyHelper(PlexusContainer container, MavenProject mavenProject, MavenSession mavenSession, Log log) throws MojoExecutionException {
+ public static DependencyHelper createDependencyHelper(PlexusContainer container, MavenProject mavenProject, MavenSession mavenSession, int cacheSize, Log log) throws MojoExecutionException {
try {
if (container.hasComponent("org.sonatype.aether.RepositorySystem")) {
org.sonatype.aether.RepositorySystem system = container.lookup(org.sonatype.aether.RepositorySystem.class);
@@ -65,7 +65,7 @@ public class DependencyHelperFactory {
throw new MojoExecutionException(e.getMessage(), e);
}
List<?> repositories = mavenProject.getRemoteProjectRepositories();
- return new Dependency31Helper(repositories, session, system);
+ return new Dependency31Helper(repositories, session, system, cacheSize);
}
} catch (ComponentLookupException e) {
throw new MojoExecutionException(e.getMessage(), e);
@@ -73,4 +73,4 @@ public class DependencyHelperFactory {
throw new MojoExecutionException("Cannot locate either org.sonatype.aether.RepositorySystem or org.eclipse.aether.RepositorySystem");
}
-}
+}
\ No newline at end of file
diff --git a/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/SimpleLRUCache.java b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/SimpleLRUCache.java
new file mode 100644
index 0000000..98ad90f
--- /dev/null
+++ b/tooling/karaf-maven-plugin/src/main/java/org/apache/karaf/tooling/utils/SimpleLRUCache.java
@@ -0,0 +1,45 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.karaf.tooling.utils;
+
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+
+/**
+ * A very simplistic LRU cache based on LinkedHashMap. It grows up to the size specified in the constructor,
+ * evicting least recently accessed entries to keep the size there.
+ */
+public final class SimpleLRUCache<K, V> extends LinkedHashMap<K, V> {
+ private static final long serialVersionUID = 1L;
+
+ private final int maxEntries;
+
+ public SimpleLRUCache(int maxEntries) {
+ this(16, 0.75f, maxEntries);
+ }
+
+ public SimpleLRUCache(int initialSize, float loadFactor, int maxEntries) {
+ super(initialSize, loadFactor, true);
+ this.maxEntries = maxEntries;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Entry<K, V> eldest) {
+ return size() > maxEntries;
+ }
+}
--
To stop receiving notification emails like this one, please contact
jbonofre@apache.org.