You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2023/07/06 17:42:54 UTC
[camel] branch main updated: CAMEL-19585: camel-jbang - Keep track of what JARs was downloaded from remote Maven repo and output in log / dev-console. This makes it easier to see what dependencies was dynamic downloaded (and from where).
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 6f4d78a7374 CAMEL-19585: camel-jbang - Keep track of what JARs was downloaded from remote Maven repo and output in log / dev-console. This makes it easier to see what dependencies was dynamic downloaded (and from where).
6f4d78a7374 is described below
commit 6f4d78a7374a18c6d1f4d25d73abc3dfe3dd550b
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Thu Jul 6 19:42:12 2023 +0200
CAMEL-19585: camel-jbang - Keep track of what JARs was downloaded from remote Maven repo and output in log / dev-console. This makes it easier to see what dependencies was dynamic downloaded (and from where).
---
.../main/console/DependencyDownloaderConsole.java | 30 ++++++++++++++++++
.../camel/main/download/DependencyDownloader.java | 14 ++++++++
.../download/DependencyDownloaderClassLoader.java | 4 ++-
...nloaderClassLoader.java => DownloadRecord.java} | 36 ++++++---------------
.../camel/main/download/DownloadThreadPool.java | 32 ++++++++++++++-----
.../main/download/MavenDependencyDownloader.java | 21 +++++++++++-
.../camel/tooling/maven/MavenDownloader.java | 27 +++++++---------
.../camel/tooling/maven/MavenDownloaderImpl.java | 37 ++++++++++++++++++++++
.../org/apache/camel/tooling/maven/MavenGav.java | 2 +-
.../maven/RemoteArtifactDownloadListener.java | 29 +++++++++++++++++
10 files changed, 180 insertions(+), 52 deletions(-)
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/console/DependencyDownloaderConsole.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/console/DependencyDownloaderConsole.java
index 3420429b2e6..76b5387d816 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/console/DependencyDownloaderConsole.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/console/DependencyDownloaderConsole.java
@@ -19,8 +19,12 @@ package org.apache.camel.main.console;
import java.util.Map;
import org.apache.camel.main.download.DependencyDownloaderClassLoader;
+import org.apache.camel.main.download.DownloadRecord;
+import org.apache.camel.main.download.MavenDependencyDownloader;
import org.apache.camel.spi.annotations.DevConsole;
import org.apache.camel.support.console.AbstractDevConsole;
+import org.apache.camel.util.TimeUtils;
+import org.apache.camel.util.json.JsonArray;
import org.apache.camel.util.json.JsonObject;
@DevConsole("dependency-downloader")
@@ -43,6 +47,16 @@ public class DependencyDownloaderConsole extends AbstractDevConsole {
sb.append("\n ").append(cp).append("\n");
}
+ MavenDependencyDownloader downloader = getCamelContext().hasService(MavenDependencyDownloader.class);
+ if (downloader != null) {
+ sb.append("\nDownloads:");
+ for (DownloadRecord r : downloader.downloadRecords()) {
+ sb.append("\n ").append(String.format("%s:%s:%s (took: %s) from: %s@%s",
+ r.groupId(), r.artifactId(), r.version(), TimeUtils.printDuration(r.elapsed(), true), r.repoId(),
+ r.repoUrl()));
+ }
+ }
+
return sb.toString();
}
@@ -57,6 +71,22 @@ public class DependencyDownloaderConsole extends AbstractDevConsole {
root.put("dependencies", cp);
}
+ MavenDependencyDownloader downloader = getCamelContext().hasService(MavenDependencyDownloader.class);
+ if (downloader != null) {
+ JsonArray arr = new JsonArray();
+ root.put("downloads", arr);
+ for (DownloadRecord r : downloader.downloadRecords()) {
+ JsonObject jo = new JsonObject();
+ arr.add(jo);
+ jo.put("groupId", r.groupId());
+ jo.put("artifactId", r.artifactId());
+ jo.put("version", r.version());
+ jo.put("elapsed", r.elapsed());
+ jo.put("repoId", r.repoId());
+ jo.put("repoUrl", r.repoUrl());
+ }
+ }
+
return root;
}
}
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloader.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloader.java
index 131d6ae2de5..15bbef3c70f 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloader.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloader.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.main.download;
+import java.util.Collection;
import java.util.List;
import org.apache.camel.CamelContextAware;
@@ -160,4 +161,17 @@ public interface DependencyDownloader extends CamelContextAware, StaticService {
*/
void onLoadingModeline(String key, String value);
+ /**
+ * Gets download record for a given artifact
+ *
+ * @return download record (if any) or <tt>null</tt> if artifact was not downloaded, but could have been resolved
+ * from local disk
+ */
+ DownloadRecord getDownloadState(String groupId, String artifactId, String version);
+
+ /**
+ * Gets the records for the downloaded artifacts
+ */
+ Collection<DownloadRecord> downloadRecords();
+
}
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java
index b01ab78a4eb..119248c2b5c 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
+import org.apache.camel.util.FileUtil;
+
public class DependencyDownloaderClassLoader extends URLClassLoader {
private static final URL[] EMPTY_URL_ARRAY = new URL[0];
@@ -41,6 +43,6 @@ public class DependencyDownloaderClassLoader extends URLClassLoader {
}
public List<String> getDownloaded() {
- return Arrays.stream(getURLs()).map(URL::getFile).collect(Collectors.toList());
+ return Arrays.stream(getURLs()).map(u -> FileUtil.stripPath(u.getFile())).collect(Collectors.toList());
}
}
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadRecord.java
similarity index 50%
copy from dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java
copy to dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadRecord.java
index b01ab78a4eb..db6b2fa3c66 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DependencyDownloaderClassLoader.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadRecord.java
@@ -16,31 +16,15 @@
*/
package org.apache.camel.main.download;
-import java.io.File;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class DependencyDownloaderClassLoader extends URLClassLoader {
-
- private static final URL[] EMPTY_URL_ARRAY = new URL[0];
-
- public DependencyDownloaderClassLoader(ClassLoader parent) {
- super(EMPTY_URL_ARRAY, parent);
- }
-
- public void addFile(File file) {
- try {
- super.addURL(file.toURI().toURL());
- } catch (MalformedURLException e) {
- throw new DownloadException("Error adding JAR to classloader: " + file, e);
- }
- }
+/**
+ * Record for details when an artifact was downloaded from a remote Maven repository.
+ */
+public record DownloadRecord(
+ String groupId,
+ String artifactId,
+ String version,
+ String repoId,
+ String repoUrl,
+ long elapsed) {
- public List<String> getDownloaded() {
- return Arrays.stream(getURLs()).map(URL::getFile).collect(Collectors.toList());
- }
}
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadThreadPool.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadThreadPool.java
index 63797507274..0a29bf8b101 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadThreadPool.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/DownloadThreadPool.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeoutException;
import org.apache.camel.CamelContext;
import org.apache.camel.CamelContextAware;
import org.apache.camel.support.service.ServiceSupport;
+import org.apache.camel.tooling.maven.MavenGav;
import org.apache.camel.util.StopWatch;
import org.apache.camel.util.TimeUtils;
import org.slf4j.Logger;
@@ -34,9 +35,14 @@ import org.slf4j.Logger;
*/
class DownloadThreadPool extends ServiceSupport implements CamelContextAware {
+ private final MavenDependencyDownloader downloader;
private CamelContext camelContext;
private volatile ExecutorService executorService;
+ public DownloadThreadPool(MavenDependencyDownloader downloader) {
+ this.downloader = downloader;
+ }
+
@Override
public CamelContext getCamelContext() {
return camelContext;
@@ -70,15 +76,25 @@ class DownloadThreadPool extends ServiceSupport implements CamelContextAware {
}
}
- // only report at INFO if downloading took > 1s because loading from cache is faster
- // and then it is not downloaded over the internet
- long taken = watch.taken();
- String msg = "Downloaded: " + gav + " (took: "
- + TimeUtils.printDuration(taken, true) + ")";
- if (taken < 1000) {
- log.debug(msg);
- } else {
+ MavenGav a = MavenGav.parseGav(gav);
+ DownloadRecord record = downloader.getDownloadState(a.getGroupId(), a.getArtifactId(), a.getVersion());
+ if (record != null) {
+ long taken = watch.taken();
+ String url = record.repoUrl();
+ String id = record.repoId();
+ String msg = "Downloaded: " + gav + " (took: "
+ + TimeUtils.printDuration(taken, true) + ") from: " + id + "@" + url;
log.info(msg);
+ } else {
+ long taken = watch.taken();
+ String msg = "Resolved: " + gav + " (took: "
+ + TimeUtils.printDuration(taken, true) + ")";
+ if (taken > 2000) {
+ // slow resolving then log
+ log.info(msg);
+ } else {
+ log.debug(msg);
+ }
}
}
diff --git a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/MavenDependencyDownloader.java b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/MavenDependencyDownloader.java
index 2f0abc589c4..164701bdb20 100644
--- a/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/MavenDependencyDownloader.java
+++ b/dsl/camel-kamelet-main/src/main/java/org/apache/camel/main/download/MavenDependencyDownloader.java
@@ -23,9 +23,12 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@ -63,6 +66,7 @@ public class MavenDependencyDownloader extends ServiceSupport implements Depende
private CamelContext camelContext;
private final Set<DownloadListener> downloadListeners = new LinkedHashSet<>();
private final Set<ArtifactDownloadListener> artifactDownloadListeners = new LinkedHashSet<>();
+ private final Map<String, DownloadRecord> downloadRecords = new HashMap<>();
private KnownReposResolver knownReposResolver;
// all maven-resolver work is delegated to camel-tooling-maven
@@ -394,6 +398,16 @@ public class MavenDependencyDownloader extends ServiceSupport implements Depende
}
}
+ @Override
+ public DownloadRecord getDownloadState(String groupId, String artifactId, String version) {
+ return downloadRecords.get(groupId + ":" + artifactId + ":" + version);
+ }
+
+ @Override
+ public Collection<DownloadRecord> downloadRecords() {
+ return downloadRecords.values();
+ }
+
private Set<String> resolveExtraRepositories(String repositoryList) {
Set<String> repositories = new LinkedHashSet<>();
if (repositoryList != null) {
@@ -417,7 +431,7 @@ public class MavenDependencyDownloader extends ServiceSupport implements Depende
if (classLoader == null && camelContext != null) {
classLoader = camelContext.getApplicationContextClassLoader();
}
- threadPool = new DownloadThreadPool();
+ threadPool = new DownloadThreadPool(this);
threadPool.setCamelContext(camelContext);
ServiceHelper.buildService(threadPool);
@@ -426,6 +440,11 @@ public class MavenDependencyDownloader extends ServiceSupport implements Depende
mavenDownloaderImpl.setMavenSettingsSecurityLocation(mavenSettingsSecurity);
mavenDownloaderImpl.setRepos(repos);
mavenDownloaderImpl.setFresh(fresh);
+ // use listener to keep track of which JARs was downloaded from a remote Maven repo (and how long time it took)
+ mavenDownloaderImpl.setRemoteArtifactDownloadListener((groupId, artifactId, version, repoId, repoUrl, elapsed) -> {
+ String gav = groupId + ":" + artifactId + ":" + version;
+ downloadRecords.put(gav, new DownloadRecord(groupId, artifactId, version, repoId, repoUrl, elapsed));
+ });
ServiceHelper.buildService(mavenDownloaderImpl);
mavenDownloader = mavenDownloaderImpl;
diff --git a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloader.java b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloader.java
index b0a4f2eb704..1d9def72135 100644
--- a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloader.java
+++ b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloader.java
@@ -29,12 +29,10 @@ public interface MavenDownloader {
* {@code groupId:artifactId[:packaging[:classifier]]:version}) is used to download artifacts from configured Maven
* repositories.
*
- * @param dependencyGAVs a list of Maven coordinates
- * @param extraRepositories nullable list of additional repositories to use (except the discovered ones)
- * @param transitively whether to download/resolve dependencies transitively
- * @param useApacheSnapshots whether to include Apache Snapshots repository in the list of used repositories
- * @return
- * @throws {@link MavenResolutionException} that can hold a list of repositories used during resolution.
+ * @param dependencyGAVs a list of Maven coordinates
+ * @param extraRepositories nullable list of additional repositories to use (except the discovered ones)
+ * @param transitively whether to download/resolve dependencies transitively
+ * @param useApacheSnapshots whether to include Apache Snapshots repository in the list of used repositories
*/
List<MavenArtifact> resolveArtifacts(
List<String> dependencyGAVs, Set<String> extraRepositories,
@@ -44,10 +42,9 @@ public interface MavenDownloader {
/**
* Resolves available versions for groupId + artifactId from single remote repository.
*
- * @param groupId
- * @param artifactId
- * @param repository external repository to use (defaults to Maven Central if {@code null})
- * @return
+ * @param groupId groupId
+ * @param artifactId artifactId
+ * @param repository external repository to use (defaults to Maven Central if {@code null})
*/
List<MavenGav> resolveAvailableVersions(String groupId, String artifactId, String repository)
throws MavenResolutionException;
@@ -56,12 +53,12 @@ public interface MavenDownloader {
* Existing, configured {@link MavenDownloader} can be used as a template to create customized version which shares
* most of the configuration except underlying {@code org.eclipse.aether.RepositorySystemSession}, which can't be
* shared.
- *
- * @param localRepository
- * @param connectTimeout
- * @param requestTimeout
- * @return
*/
MavenDownloader customize(String localRepository, int connectTimeout, int requestTimeout);
+ /**
+ * To use a listener when downloading from remote repositories.
+ */
+ void setRemoteArtifactDownloadListener(RemoteArtifactDownloadListener listener);
+
}
diff --git a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloaderImpl.java b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloaderImpl.java
index 3340e24fbb5..7c72d6d821c 100644
--- a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloaderImpl.java
+++ b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenDownloaderImpl.java
@@ -39,6 +39,7 @@ import java.util.stream.Collectors;
import org.apache.camel.support.service.ServiceSupport;
import org.apache.camel.tooling.maven.support.DIRegistry;
+import org.apache.camel.util.StopWatch;
import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader;
import org.apache.maven.model.building.DefaultModelBuilderFactory;
import org.apache.maven.model.building.ModelBuilder;
@@ -71,8 +72,10 @@ import org.apache.maven.settings.io.SettingsWriter;
import org.apache.maven.settings.validation.DefaultSettingsValidator;
import org.apache.maven.settings.validation.SettingsValidator;
import org.codehaus.plexus.util.xml.Xpp3Dom;
+import org.eclipse.aether.AbstractRepositoryListener;
import org.eclipse.aether.ConfigurationProperties;
import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositoryEvent;
import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
@@ -150,6 +153,7 @@ import org.eclipse.aether.named.providers.FileLockNamedLockFactory;
import org.eclipse.aether.named.providers.LocalReadWriteLockNamedLockFactory;
import org.eclipse.aether.named.providers.LocalSemaphoreNamedLockFactory;
import org.eclipse.aether.named.providers.NoopNamedLockFactory;
+import org.eclipse.aether.repository.ArtifactRepository;
import org.eclipse.aether.repository.Authentication;
import org.eclipse.aether.repository.AuthenticationSelector;
import org.eclipse.aether.repository.LocalRepository;
@@ -252,6 +256,7 @@ public class MavenDownloaderImpl extends ServiceSupport implements MavenDownload
// comma-separated list of additional repositories to use
private String repos;
private boolean fresh;
+ private RemoteArtifactDownloadListener remoteArtifactDownloadListener;
private boolean apacheSnapshotsIncluded;
@@ -321,6 +326,11 @@ public class MavenDownloaderImpl extends ServiceSupport implements MavenDownload
}
}
+ @Override
+ public void setRemoteArtifactDownloadListener(RemoteArtifactDownloadListener remoteArtifactDownloadListener) {
+ this.remoteArtifactDownloadListener = remoteArtifactDownloadListener;
+ }
+
@Override
public List<MavenArtifact> resolveArtifacts(
List<String> dependencyGAVs, Set<String> extraRepositories,
@@ -361,6 +371,33 @@ public class MavenDownloaderImpl extends ServiceSupport implements MavenDownload
//collectRequest.addManagedDependency(...);
}
+ if (remoteArtifactDownloadListener != null && repositorySystemSession instanceof DefaultRepositorySystemSession) {
+ DefaultRepositorySystemSession drss = (DefaultRepositorySystemSession) repositorySystemSession;
+ drss.setRepositoryListener(new AbstractRepositoryListener() {
+ private final StopWatch watch = new StopWatch();
+
+ @Override
+ public void artifactDownloading(RepositoryEvent event) {
+ watch.restart();
+ }
+
+ @Override
+ public void artifactDownloaded(RepositoryEvent event) {
+ if (event.getArtifact() != null) {
+ Artifact a = event.getArtifact();
+
+ ArtifactRepository ar = event.getRepository();
+ String url = ar instanceof RemoteRepository ? ((RemoteRepository) ar).getUrl() : null;
+ String id = ar != null ? ar.getId() : null;
+ long elapsed = watch.takenAndRestart();
+ String version = a.isSnapshot() ? a.getBaseVersion() : a.getVersion();
+ remoteArtifactDownloadListener.artifactDownloaded(a.getGroupId(), a.getArtifactId(), version,
+ id, url, elapsed);
+ }
+ }
+ });
+ }
+
if (transitively) {
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, new AcceptAllDependencyFilter());
try {
diff --git a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenGav.java b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenGav.java
index e28fec5ed37..3e0127b0c9d 100644
--- a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenGav.java
+++ b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/MavenGav.java
@@ -17,7 +17,7 @@
package org.apache.camel.tooling.maven;
/**
- * Maven GAV model with parsing support and speacial rules for some names:
+ * Maven GAV model with parsing support and special rules for some names:
* <ul>
* <li>{@code camel:core -> org.apache.camel:camel-core}</li>
* <li>{@code camel-xxx -> org.apache.camel:camel-xxx}</li>
diff --git a/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/RemoteArtifactDownloadListener.java b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/RemoteArtifactDownloadListener.java
new file mode 100644
index 00000000000..2e86270c9ec
--- /dev/null
+++ b/tooling/camel-tooling-maven/src/main/java/org/apache/camel/tooling/maven/RemoteArtifactDownloadListener.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.tooling.maven;
+
+@FunctionalInterface
+public interface RemoteArtifactDownloadListener {
+
+ /**
+ * Event when an artifact was downloaded from a remote maven repository (not local).
+ */
+ void artifactDownloaded(
+ String groupId, String artifactId, String version,
+ String repoId, String repoUrl, long elapsed);
+
+}