You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2024/03/21 13:34:34 UTC
(camel) branch main updated: Validate routes in transitive dependencies (#13567)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 7ce99c5153f Validate routes in transitive dependencies (#13567)
7ce99c5153f is described below
commit 7ce99c5153f5f454563b9268a139c480d88d08af
Author: Federico Mariani <34...@users.noreply.github.com>
AuthorDate: Thu Mar 21 14:34:27 2024 +0100
Validate routes in transitive dependencies (#13567)
Add docs and logs
Add docs
---
.../src/main/docs/camel-report-maven-plugin.adoc | 34 +++++
.../java/org/apache/camel/maven/ValidateMojo.java | 144 +++++++++++++++++++--
2 files changed, 166 insertions(+), 12 deletions(-)
diff --git a/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc b/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc
index 9da9525fcf2..c0cf076b7c2 100644
--- a/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc
+++ b/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc
@@ -151,6 +151,9 @@ The maven plugin *validate* goal supports the following options which can be con
| directOrSedaPairCheck | true |Whether to validate direct/seda endpoints sending to non existing consumers.
| configurationFiles | application.properties | Location of configuration files to validate. The default is application.properties. Multiple values can be separated by comma and use wildcard pattern matching.
| showAll | false | Whether to show all endpoints and simple expressions (both invalid and valid).
+| downloadTransitiveArtifacts | false | When sourcesArtifacts are declared, this flag can be used to download transitive dependencies, carefully enable this flag since it will try to download the whole dependency tree.
+| sourcesArtifacts | | List of sources transitive dependencies that contain camel routes, this option can be used to download extra dependencies that contain camel route that your project may depend on.
+| extraMavenRepositories | | List of extra maven repositories.
|===
For example to turn on ignoring usage of deprecated options from the command line, you can run:
@@ -171,6 +174,37 @@ $cd myproject
$mvn org.apache.camel:camel-report-maven-plugin:3.0.0:validate -Dcamel.includeTest=true
----
+=== Validate Apache Camel routes in transitive dependencies
+
+If your routes use `direct` or `seda` endpoints that are not present in the current project, but the routes are declared into a dependency of your project, you can edit the plugin configuration accordingly so that these routes can be taken into account by the Camel validate plugin.
+In particular, in order to use the validate plugin with transitive dependencies, *sources jars are needed*, for example:
+
+* Given the following Camel route `from("direct:in").to("direct:out")` defined in the current project
+* The route `from("direct:out")` is declared into a dependency of your project, for example `my.company:routes-dependency:1.0`
+* If `routes-dependency` sources are released into a maven repository, the following plugin configuration can be used:
+
+```xml
+<plugin>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-report-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>validate</goal>
+ </goals>
+ <configuration>
+ <sourcesArtifacts>
+ <sourcesArtifact>my.company:routes-dependency:jar:sources:1.0</sourcesArtifact>
+ </sourcesArtifacts>
+ <extraMavenRepositories>
+ <extraMavenRepository>http://internal.repo:8080/maven</extraMavenRepository>
+ </extraMavenRepositories>
+ </configuration>
+ </execution>
+ </executions>
+</plugin>
+```
== camel-report:route-coverage
diff --git a/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java b/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java
index 0a7fc18c66d..0e7436ff431 100644
--- a/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java
+++ b/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java
@@ -16,23 +16,21 @@
*/
package org.apache.camel.maven;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.InputStream;
-import java.io.LineNumberReader;
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import org.apache.camel.catalog.CamelCatalog;
import org.apache.camel.catalog.ConfigurationPropertiesValidationResult;
import org.apache.camel.catalog.DefaultCamelCatalog;
import org.apache.camel.catalog.EndpointValidationResult;
import org.apache.camel.catalog.LanguageValidationResult;
+import org.apache.camel.catalog.common.FileUtil;
import org.apache.camel.catalog.lucene.LuceneSuggestionStrategy;
import org.apache.camel.catalog.maven.MavenVersionManager;
import org.apache.camel.parser.RouteBuilderParser;
@@ -41,8 +39,12 @@ import org.apache.camel.parser.model.CamelEndpointDetails;
import org.apache.camel.parser.model.CamelRouteDetails;
import org.apache.camel.parser.model.CamelSimpleExpressionDetails;
import org.apache.camel.support.PatternHelper;
+import org.apache.camel.tooling.maven.MavenArtifact;
+import org.apache.camel.tooling.maven.MavenDownloaderImpl;
+import org.apache.camel.tooling.maven.MavenResolutionException;
import org.apache.camel.util.OrderedProperties;
import org.apache.camel.util.StringHelper;
+import org.apache.commons.io.IOUtils;
import org.apache.maven.model.Dependency;
import org.apache.maven.model.Resource;
import org.apache.maven.plugin.MojoExecutionException;
@@ -95,6 +97,21 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(property = "camel.includeJava", defaultValue = "true")
private boolean includeJava;
+ /**
+ * List of extra maven repositories
+ */
+ @Parameter(property = "camel.extraRepositories")
+ private String[] extraMavenRepositories;
+
+ /**
+ * List of sources transitive dependencies that contains camel routes
+ */
+ @Parameter(property = "camel.sourcesArtifacts")
+ private String[] sourcesArtifacts;
+
+ @Parameter(defaultValue = "${project.build.directory}")
+ private String projectBuildDir;
+
/**
* Whether to include XML files to be validated for invalid Camel endpoints
*/
@@ -173,6 +190,13 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(property = "camel.directOrSedaPairCheck", defaultValue = "true")
private boolean directOrSedaPairCheck;
+ /**
+ * When sourcesArtifacts are declared, choose to download transitive artifacts or not carefully enable this flag
+ * since it will try to download the whole dependency tree
+ */
+ @Parameter(property = "camel.downloadTransitiveArtifacts", defaultValue = "false")
+ private boolean downloadTransitiveArtifacts;
+
/**
* Location of configuration files to validate. The default is application.properties Multiple values can be
* separated by comma and use wildcard pattern matching.
@@ -186,8 +210,87 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(defaultValue = "${repositorySystemSession}", readonly = true)
private RepositorySystemSession repositorySystemSession;
+ /**
+ * javaFiles in memory cache, useful for multi modules maven project
+ */
+ private static Set<File> javaFiles = new LinkedHashSet<>();
+ /**
+ * xmlFiles in memory cache, useful for multi modules maven project
+ */
+ private static Set<File> xmlFiles = new LinkedHashSet<>();
+
+ private static Set<String> downloadedArtifacts = new LinkedHashSet<>();
+
@Override
public void execute() throws MojoExecutionException {
+ // Download extra sources only if artifacts sources are defined and the current project is not a parent project
+ if (!"pom".equals(project.getPackaging()) && sourcesArtifacts != null && sourcesArtifacts.length > 0) {
+ // setup MavenDownloader, it will be used to download and locate artifacts declared via sourcesArtifacts
+
+ List<String> artifacts = Arrays.asList(sourcesArtifacts);
+
+ artifacts
+ .parallelStream()
+ .forEach(artifact -> {
+ if (!artifact.contains(":sources:")) {
+ getLog().warn("The artifact " + artifact
+ + " does not contain sources classifier, and may be excluded in future releases");
+ }
+ });
+
+ try (MavenDownloaderImpl downloader
+ = new MavenDownloaderImpl(repositorySystem, repositorySystemSession, getSession().getSettings())) {
+ downloader.init();
+ Set<String> repositorySet = Arrays.stream(extraMavenRepositories)
+ .collect(Collectors.toSet());
+ List<String> artifactList = new ArrayList<>(artifacts);
+
+ // Remove already downloaded Artifacts
+ artifactList.removeAll(downloadedArtifacts);
+
+ if (!artifactList.isEmpty()) {
+ getLog().info("Downloading the following artifacts: " + artifactList);
+ List<MavenArtifact> mavenSourcesArtifacts
+ = downloader.resolveArtifacts(artifactList, repositorySet, downloadTransitiveArtifacts, false);
+
+ // Create folder into the target folder that will be used to unzip
+ // the downloaded artifacts
+ Path extraSourcesPath = Paths.get(projectBuildDir, "camel-validate-sources");
+ if (!Files.exists(extraSourcesPath)) {
+ Files.createDirectories(extraSourcesPath);
+ }
+
+ // Unzip all the downloaded artifacts and add javas and xmls files into the cache
+ for (MavenArtifact artifact : mavenSourcesArtifacts) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(artifact.getGav().getGroupId()).append(":")
+ .append(artifact.getGav().getArtifactId()).append(":")
+ .append(artifact.getGav().getPackaging()).append(":")
+ .append(artifact.getGav().getClassifier()).append(":")
+ .append(artifact.getGav().getVersion());
+ // Avoid downloading the same dependency multiple times
+ downloadedArtifacts.add(sb.toString());
+
+ Path target = extraSourcesPath.resolve(artifact.getGav().getArtifactId());
+ getLog().info("Unzipping the artifact: " + artifact + " to " + target);
+ if (Files.exists(target)) {
+ continue;
+ }
+
+ unzipArtifact(artifact, target);
+
+ FileUtil.findJavaFiles(target.toFile(), javaFiles);
+ FileUtil.findXmlFiles(target.toFile(), xmlFiles);
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException(e);
+ } catch (MavenResolutionException e) {
+ // missing artifact, log and proceed
+ getLog().warn(e.getMessage());
+ }
+ }
+
CamelCatalog catalog = new DefaultCamelCatalog();
// add activemq as known component
catalog.addComponent("activemq", "org.apache.activemq.camel.component.ActiveMQComponent");
@@ -229,6 +332,25 @@ public class ValidateMojo extends AbstractExecMojo {
doExecuteConfigurationFiles(catalog);
}
+ private static void unzipArtifact(MavenArtifact artifact, Path target) throws IOException {
+ try (ZipFile zipFile = new ZipFile(artifact.getFile().toPath().toFile())) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ File entryDestination = new File(target.toString(), entry.getName());
+ if (entry.isDirectory()) {
+ entryDestination.mkdirs();
+ } else {
+ entryDestination.getParentFile().mkdirs();
+ try (InputStream in = zipFile.getInputStream(entry);
+ OutputStream out = new FileOutputStream(entryDestination)) {
+ IOUtils.copy(in, out);
+ }
+ }
+ }
+ }
+ }
+
protected void doExecuteConfigurationFiles(CamelCatalog catalog) throws MojoExecutionException {
Set<File> propertiesFiles = new LinkedHashSet<>();
for (Resource dir : project.getResources()) {
@@ -390,8 +512,6 @@ public class ValidateMojo extends AbstractExecMojo {
List<CamelEndpointDetails> endpoints = new ArrayList<>();
List<CamelSimpleExpressionDetails> simpleExpressions = new ArrayList<>();
List<CamelRouteDetails> routeIds = new ArrayList<>();
- Set<File> javaFiles = new LinkedHashSet<>();
- Set<File> xmlFiles = new LinkedHashSet<>();
// find all java route builder classes
findJavaRouteBuilderClasses(javaFiles, includeJava, includeTest, project);