You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2022/07/07 13:22:34 UTC
[flink] branch master updated: [FLINK-28201][ci] Generalize utils around dependency plugin
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 81c1cb82afc [FLINK-28201][ci] Generalize utils around dependency plugin
81c1cb82afc is described below
commit 81c1cb82afc7c1ef66a5dd21555997abaf4a7817
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Jul 7 15:22:27 2022 +0200
[FLINK-28201][ci] Generalize utils around dependency plugin
---
.../tools/ci/licensecheck/NoticeFileChecker.java | 65 ++++---
.../tools/ci/suffixcheck/ScalaSuffixChecker.java | 81 ++++-----
.../tools/ci/utils/dependency/Dependency.java | 114 ++++++++++++
.../ci/utils/dependency/DependencyParser.java | 201 +++++++++++++++++++++
.../utils/dependency/DependencyParserCopyTest.java | 109 +++++++++++
.../utils/dependency/DependencyParserTreeTest.java | 134 ++++++++++++++
6 files changed, 626 insertions(+), 78 deletions(-)
diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
index 6237ec7ec68..3dedaab8766 100644
--- a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
+++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
@@ -18,6 +18,9 @@
package org.apache.flink.tools.ci.licensecheck;
+import org.apache.flink.tools.ci.utils.dependency.Dependency;
+import org.apache.flink.tools.ci.utils.dependency.DependencyParser;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
@@ -60,13 +63,6 @@ public class NoticeFileChecker {
private static final Pattern SHADE_INCLUDE_MODULE_PATTERN =
Pattern.compile(".*Including ([^:]+):([^:]+):jar:([^ ]+) in the shaded jar");
- // pattern for maven-dependency-plugin copyied dependencies
- private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN =
- Pattern.compile(
- ".*maven-dependency-plugin:[^:]+:copy \\([^)]+\\) @ ([^ _]+)(_[0-9.]+)? --.*");
- private static final Pattern DEPENDENCY_COPY_INCLUDE_MODULE_PATTERN =
- Pattern.compile(".*Configured Artifact: ([^:]+):([^:]+):([^:]+):jar.*");
-
// Examples:
// "- org.apache.htrace:htrace-core:3.1.0-incubating"
// or
@@ -78,11 +74,14 @@ public class NoticeFileChecker {
static int run(File buildResult, Path root) throws IOException {
int severeIssueCount = 0;
// parse included dependencies from build output
- Multimap<String, IncludedDependency> modulesWithBundledDependencies =
- parseModulesFromBuildResult(buildResult);
+ final Multimap<String, IncludedDependency> modulesWithBundledDependencies =
+ combine(
+ parseModulesFromBuildResult(buildResult),
+ DependencyParser.parseDependencyCopyOutput(buildResult.toPath()));
+
LOG.info(
"Extracted "
- + modulesWithBundledDependencies.asMap().keySet().size()
+ + modulesWithBundledDependencies.keySet().size()
+ " modules with a total of "
+ modulesWithBundledDependencies.values().size()
+ " dependencies");
@@ -102,6 +101,28 @@ public class NoticeFileChecker {
return severeIssueCount;
}
+ private static Multimap<String, IncludedDependency> combine(
+ Multimap<String, IncludedDependency> modulesWithBundledDependencies,
+ Map<String, Set<Dependency>> modulesWithCopiedDependencies) {
+ modulesWithCopiedDependencies.forEach(
+ (module, copiedDependencies) -> {
+ copiedDependencies.stream()
+ .filter(
+ dependency ->
+ !dependency.getGroupId().contains("org.apache.flink"))
+ .map(
+ dependency ->
+ IncludedDependency.create(
+ dependency.getGroupId(),
+ dependency.getArtifactId(),
+ dependency.getVersion()))
+ .forEach(
+ dependency ->
+ modulesWithBundledDependencies.put(module, dependency));
+ });
+ return modulesWithBundledDependencies;
+ }
+
private static int ensureRequiredNoticeFiles(
Multimap<String, IncludedDependency> modulesWithShadedDependencies,
List<Path> noticeFiles) {
@@ -274,19 +295,12 @@ public class NoticeFileChecker {
try (Stream<String> lines = Files.lines(buildResult.toPath())) {
// String line;
String currentShadeModule = null;
- String currentDependencyCopyModule = null;
for (String line : (Iterable<String>) lines::iterator) {
Matcher nextShadeModuleMatcher = SHADE_NEXT_MODULE_PATTERN.matcher(line);
if (nextShadeModuleMatcher.find()) {
currentShadeModule = nextShadeModuleMatcher.group(2);
}
- Matcher nextDependencyCopyModuleMatcher =
- DEPENDENCY_COPY_NEXT_MODULE_PATTERN.matcher(line);
- if (nextDependencyCopyModuleMatcher.find()) {
- currentDependencyCopyModule = nextDependencyCopyModuleMatcher.group(1);
- }
-
if (currentShadeModule != null) {
Matcher includeMatcher = SHADE_INCLUDE_MODULE_PATTERN.matcher(line);
if (includeMatcher.find()) {
@@ -300,26 +314,9 @@ public class NoticeFileChecker {
}
}
}
-
- if (currentDependencyCopyModule != null) {
- Matcher copyMatcher = DEPENDENCY_COPY_INCLUDE_MODULE_PATTERN.matcher(line);
- if (copyMatcher.find()) {
- String groupId = copyMatcher.group(1);
- String artifactId = copyMatcher.group(2);
- String version = copyMatcher.group(3);
- if (!"org.apache.flink".equals(groupId)) {
- result.put(
- currentDependencyCopyModule,
- IncludedDependency.create(groupId, artifactId, version));
- }
- }
- }
if (line.contains("Replacing original artifact with shaded artifact")) {
currentShadeModule = null;
}
- if (line.contains("Copying")) {
- currentDependencyCopyModule = null;
- }
}
}
return result;
diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
index 1a00dfb538b..7b3d3d83cc8 100644
--- a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
+++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
@@ -17,10 +17,12 @@
package org.apache.flink.tools.ci.suffixcheck;
+import org.apache.flink.tools.ci.utils.dependency.Dependency;
+import org.apache.flink.tools.ci.utils.dependency.DependencyParser;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -31,8 +33,8 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -96,49 +98,39 @@ public class ScalaSuffixChecker {
final Set<String> cleanModules = new HashSet<>();
final Set<String> infectedModules = new HashSet<>();
- try (final BufferedReader bufferedReader =
- Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
-
- String line;
- while ((line = bufferedReader.readLine()) != null) {
- final Matcher matcher = moduleNamePattern.matcher(line);
- if (matcher.matches()) {
- final String moduleName = stripScalaSuffix(matcher.group(1));
- if (isExcluded(moduleName)) {
- continue;
- }
- LOG.trace("Parsing module '{}'.", moduleName);
-
- // skip: [INFO] org.apache.flink:flink-annotations:jar:1.14-SNAPSHOT
- bufferedReader.readLine();
-
- boolean infected = false;
- line = bufferedReader.readLine();
- while (blockPattern.matcher(line).matches()) {
- final boolean dependsOnScala = dependsOnScala(line);
- final boolean isTestDependency = line.endsWith(":test");
- // we ignored flink-rpc-akka because it is loaded through a separate class
- // loader
- final boolean isExcluded = isExcluded(line);
- LOG.trace("\tline:{}", line);
- LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala);
- LOG.trace("\t\tis-test-dependency:{}", isTestDependency);
- LOG.trace("\t\tis-excluded:{}", isExcluded);
- if (dependsOnScala && !isTestDependency && !isExcluded) {
- LOG.trace("\t\tOutbreak detected at {}!", moduleName);
- infected = true;
- }
-
- line = bufferedReader.readLine();
- }
+ final Map<String, Set<Dependency>> dependenciesByModule =
+ DependencyParser.parseDependencyTreeOutput(path);
- if (infected) {
- infectedModules.add(moduleName);
- } else {
- cleanModules.add(moduleName);
- }
+ for (String module : dependenciesByModule.keySet()) {
+ final String moduleName = stripScalaSuffix(module);
+ if (isExcluded(moduleName)) {
+ continue;
+ }
+ LOG.trace("Processing module '{}'.", moduleName);
+
+ final Collection<Dependency> dependencies = dependenciesByModule.get(module);
+
+ boolean infected = false;
+ for (Dependency dependency : dependencies) {
+ final boolean dependsOnScala = dependsOnScala(dependency);
+ final boolean isTestDependency = dependency.getScope().get().equals("test");
+ // we ignored flink-rpc-akka because it is loaded through a separate class loader
+ final boolean isExcluded = isExcluded(dependency.getArtifactId());
+ LOG.trace("\tdependency:{}", dependency);
+ LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala);
+ LOG.trace("\t\tis-test-dependency:{}", isTestDependency);
+ LOG.trace("\t\tis-excluded:{}", isExcluded);
+ if (dependsOnScala && !isTestDependency && !isExcluded) {
+ LOG.trace("\t\tOutbreak detected at {}!", moduleName);
+ infected = true;
}
}
+
+ if (infected) {
+ infectedModules.add(moduleName);
+ } else {
+ cleanModules.add(moduleName);
+ }
}
return new ParseResult(cleanModules, infectedModules);
@@ -149,8 +141,9 @@ public class ScalaSuffixChecker {
return i > 0 ? moduleName.substring(0, i) : moduleName;
}
- private static boolean dependsOnScala(final String line) {
- return line.contains("org.scala-lang") || scalaSuffixPattern.matcher(line).find();
+ private static boolean dependsOnScala(final Dependency dependency) {
+ return dependency.getGroupId().contains("org.scala-lang")
+ || scalaSuffixPattern.matcher(dependency.getArtifactId()).find();
}
private static Collection<String> checkScalaSuffixes(
diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/Dependency.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/Dependency.java
new file mode 100644
index 00000000000..4f8816a35c3
--- /dev/null
+++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/Dependency.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tools.ci.utils.dependency;
+
+import javax.annotation.Nullable;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Represents a dependency.
+ *
+ * <p>For some properties we return an {@link Optional}, for those that, depending on the plugin
+ * goal, may not be determinable. For example, {@code dependency:copy} never prints the scope or
+ * optional flag.
+ */
+public final class Dependency {
+
+ private final String groupId;
+ private final String artifactId;
+ private final String version;
+ @Nullable private final String scope;
+ @Nullable private final Boolean isOptional;
+
+ private Dependency(
+ String groupId,
+ String artifactId,
+ String version,
+ @Nullable String scope,
+ @Nullable Boolean isOptional) {
+ this.groupId = Objects.requireNonNull(groupId);
+ this.artifactId = Objects.requireNonNull(artifactId);
+ this.version = Objects.requireNonNull(version);
+ this.scope = scope;
+ this.isOptional = isOptional;
+ }
+
+ public static Dependency create(
+ String groupId, String artifactId, String version, String scope, boolean isOptional) {
+ return new Dependency(
+ groupId, artifactId, version, Objects.requireNonNull(scope), isOptional);
+ }
+
+ public static Dependency create(String groupId, String artifactId, String version) {
+ return new Dependency(groupId, artifactId, version, null, null);
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public String getArtifactId() {
+ return artifactId;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public Optional<String> getScope() {
+ return Optional.ofNullable(scope);
+ }
+
+ public Optional<Boolean> isOptional() {
+ return Optional.ofNullable(isOptional);
+ }
+
+ @Override
+ public String toString() {
+ return groupId
+ + ":"
+ + artifactId
+ + ":"
+ + version
+ + (scope != null ? ":" + scope : "")
+ + (isOptional != null && isOptional ? " (optional)" : "");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Dependency that = (Dependency) o;
+ return Objects.equals(groupId, that.groupId)
+ && Objects.equals(artifactId, that.artifactId)
+ && Objects.equals(version, that.version)
+ && Objects.equals(scope, that.scope)
+ && Objects.equals(isOptional, that.isOptional);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(groupId, artifactId, version, scope, isOptional);
+ }
+}
diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/DependencyParser.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/DependencyParser.java
new file mode 100644
index 00000000000..dff24888c32
--- /dev/null
+++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/utils/dependency/DependencyParser.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tools.ci.utils.dependency;
+
+import org.apache.flink.annotation.VisibleForTesting;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+/** Parsing utils for the Maven dependency plugin. */
+public class DependencyParser {
+
+ private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN =
+ Pattern.compile(".*maven-dependency-plugin:[^:]+:copy .* @ ([^ _]+)(_[0-9.]+)? --.*");
+
+ private static final Pattern DEPENDENCY_TREE_NEXT_MODULE_PATTERN =
+ Pattern.compile(".*maven-dependency-plugin:[^:]+:tree .* @ ([^ _]+)(_[0-9.]+)? --.*");
+
+ /** See {@link DependencyParserTreeTest} for examples. */
+ private static final Pattern DEPENDENCY_TREE_ITEM_PATTERN =
+ Pattern.compile(
+ ".* +"
+ + "(?<groupId>.*?):"
+ + "(?<artifactId>.*?):"
+ + "(?<type>.*?):"
+ + "(?:(?<classifier>.*?):)?"
+ + "(?<version>.*?):"
+ + "(?<scope>[^ ]*)"
+ + "(?<optional> \\(optional\\))?");
+
+ /** See {@link DependencyParserCopyTest} for examples. */
+ private static final Pattern DEPENDENCY_COPY_ITEM_PATTERN =
+ Pattern.compile(
+ ".* Configured Artifact: +"
+ + "(?<groupId>.*?):"
+ + "(?<artifactId>.*?):"
+ + "(?:(?<classifier>.*?):)?"
+ + "(?<version>.*?):"
+ + "(?:\\?:)?" // unknown cause; e.g.: javax.xml.bind:jaxb-api:?:jar
+ + "(?<type>.*)");
+
+ /**
+ * Parses the output of a Maven build where {@code dependency:copy} was used, and returns a set
+ * of copied dependencies for each module.
+ *
+ * <p>The returned dependencies will NEVER contain the scope or optional flag.
+ */
+ public static Map<String, Set<Dependency>> parseDependencyCopyOutput(Path buildOutput)
+ throws IOException {
+ return processLines(buildOutput, DependencyParser::parseDependencyCopyOutput);
+ }
+
+ /**
+ * Parses the output of a Maven build where {@code dependency:tree} was used, and returns a set
+ * of dependencies for each module.
+ */
+ public static Map<String, Set<Dependency>> parseDependencyTreeOutput(Path buildOutput)
+ throws IOException {
+ return processLines(buildOutput, DependencyParser::parseDependencyTreeOutput);
+ }
+
+ private static <X> X processLines(Path buildOutput, Function<Stream<String>, X> processor)
+ throws IOException {
+ try (Stream<String> lines = Files.lines(buildOutput)) {
+ return processor.apply(lines.filter(line -> line.contains("[INFO]")));
+ }
+ }
+
+ @VisibleForTesting
+ static Map<String, Set<Dependency>> parseDependencyCopyOutput(Stream<String> lines) {
+ return parseDependencyOutput(
+ lines,
+ DEPENDENCY_COPY_NEXT_MODULE_PATTERN,
+ DependencyParser::parseCopyDependencyBlock);
+ }
+
+ @VisibleForTesting
+ static Map<String, Set<Dependency>> parseDependencyTreeOutput(Stream<String> lines) {
+ return parseDependencyOutput(
+ lines,
+ DEPENDENCY_TREE_NEXT_MODULE_PATTERN,
+ DependencyParser::parseTreeDependencyBlock);
+ }
+
+ private static Map<String, Set<Dependency>> parseDependencyOutput(
+ Stream<String> lines,
+ Pattern executionLinePattern,
+ Function<Iterator<String>, Set<Dependency>> blockParser) {
+ final Map<String, Set<Dependency>> result = new LinkedHashMap<>();
+
+ final Iterator<String> iterator = lines.iterator();
+
+ while (iterator.hasNext()) {
+ Matcher moduleMatcher = executionLinePattern.matcher(iterator.next());
+ while (!moduleMatcher.find()) {
+ if (iterator.hasNext()) {
+ moduleMatcher = executionLinePattern.matcher(iterator.next());
+ } else {
+ return result;
+ }
+ }
+ final String currentModule = moduleMatcher.group(1);
+
+ if (!iterator.hasNext()) {
+ throw new IllegalStateException("Expected more output from the dependency-plugin.");
+ }
+
+ result.put(currentModule, blockParser.apply(iterator));
+ }
+ return result;
+ }
+
+ private static Set<Dependency> parseCopyDependencyBlock(Iterator<String> block) {
+ return parseDependencyBlock(block, DependencyParser::parseCopyDependency);
+ }
+
+ private static Set<Dependency> parseTreeDependencyBlock(Iterator<String> block) {
+ // discard one line, which only contains the current module name
+ block.next();
+
+ if (!block.hasNext()) {
+ throw new IllegalStateException("Expected more output from the dependency-plugin.");
+ }
+
+ return parseDependencyBlock(block, DependencyParser::parseTreeDependency);
+ }
+
+ private static Set<Dependency> parseDependencyBlock(
+ Iterator<String> block, Function<String, Optional<Dependency>> lineItemParser) {
+ final Set<Dependency> dependencies = new LinkedHashSet<>();
+
+ Optional<Dependency> parsedDependency = lineItemParser.apply(block.next());
+ while (parsedDependency.isPresent()) {
+ dependencies.add(parsedDependency.get());
+
+ if (block.hasNext()) {
+ parsedDependency = lineItemParser.apply(block.next());
+ } else {
+ parsedDependency = Optional.empty();
+ }
+ }
+
+ return dependencies;
+ }
+
+ @VisibleForTesting
+ static Optional<Dependency> parseCopyDependency(String line) {
+ Matcher dependencyMatcher = DEPENDENCY_COPY_ITEM_PATTERN.matcher(line);
+ if (!dependencyMatcher.find()) {
+ return Optional.empty();
+ }
+
+ return Optional.of(
+ Dependency.create(
+ dependencyMatcher.group("groupId"),
+ dependencyMatcher.group("artifactId"),
+ dependencyMatcher.group("version")));
+ }
+
+ @VisibleForTesting
+ static Optional<Dependency> parseTreeDependency(String line) {
+ Matcher dependencyMatcher = DEPENDENCY_TREE_ITEM_PATTERN.matcher(line);
+ if (!dependencyMatcher.find()) {
+ return Optional.empty();
+ }
+
+ return Optional.of(
+ Dependency.create(
+ dependencyMatcher.group("groupId"),
+ dependencyMatcher.group("artifactId"),
+ dependencyMatcher.group("version"),
+ dependencyMatcher.group("scope"),
+ dependencyMatcher.group("optional") != null));
+ }
+}
diff --git a/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java b/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java
new file mode 100644
index 00000000000..fcf682e3dae
--- /dev/null
+++ b/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tools.ci.utils.dependency;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests the parsing of {@code dependency:copy}. */
+class DependencyParserCopyTest {
+
+ private static Stream<String> getTestDependencyCopy() {
+ return Stream.of(
+ "[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m1 ---",
+ "[INFO] Configured Artifact: external:dependency1:2.1:jar",
+ "[INFO] Configured Artifact: external:dependency4:2.4:jar",
+ "[INFO] Copying dependency1-2.1.jar to /some/path/dependency1-2.1.jar",
+ "[INFO] Copying dependency4-2.4.jar to /some/path/dependency4-2.4.jar",
+ "[INFO]",
+ "[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m2 ---",
+ "[INFO] Configured Artifact: internal:m1:1.1:jar",
+ "[INFO] Copying internal-1.1.jar to /some/path/m1-1.1.jar");
+ }
+
+ @Test
+ void testCopyParsing() {
+ final Map<String, Set<Dependency>> dependenciesByModule =
+ DependencyParser.parseDependencyCopyOutput(getTestDependencyCopy());
+
+ assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2");
+ assertThat(dependenciesByModule.get("m1"))
+ .containsExactlyInAnyOrder(
+ Dependency.create("external", "dependency1", "2.1"),
+ Dependency.create("external", "dependency4", "2.4"));
+ assertThat(dependenciesByModule.get("m2"))
+ .containsExactlyInAnyOrder(Dependency.create("internal", "m1", "1.1"));
+ }
+
+ @Test
+ void testCopyLineParsingGroupId() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:jar"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.getGroupId()).isEqualTo("external"));
+ }
+
+ @Test
+ void testCopyLineParsingArtifactId() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:jar"))
+ .hasValueSatisfying(
+ dependency ->
+ assertThat(dependency.getArtifactId()).isEqualTo("dependency1"));
+ }
+
+ @Test
+ void testCopyLineParsingVersion() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:jar"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0"));
+ }
+
+ @Test
+ void testCopyLineParsingScope() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:jar"))
+ .hasValueSatisfying(dependency -> assertThat(dependency.getScope()).isEmpty());
+ }
+
+ @Test
+ void testCopyLineParsingOptional() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:jar"))
+ .hasValueSatisfying(dependency -> assertThat(dependency.isOptional()).isEmpty());
+ }
+
+ @Test
+ void testCopyLineParsingWithNonJarType() {
+ assertThat(
+ DependencyParser.parseCopyDependency(
+ "[INFO] Configured Artifact: external:dependency1:1.0:pom"))
+ .hasValue(Dependency.create("external", "dependency1", "1.0"));
+ }
+}
diff --git a/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java b/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java
new file mode 100644
index 00000000000..761fa629f0f
--- /dev/null
+++ b/tools/ci/java-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tools.ci.utils.dependency;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests the parsing of {@code dependency:tree}. */
+class DependencyParserTreeTest {
+
+ private static Stream<String> getTestDependencyTree() {
+ return Stream.of(
+ "[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m1 ---",
+ "[INFO] internal:m1:jar:1.1",
+ "[INFO] +- external:dependency1:jar:2.1:compile",
+ "[INFO] | +- external:dependency2:jar:2.2:compile (optional)",
+ "[INFO] | | \\- external:dependency3:jar:2.3:provided (optional)",
+ "[INFO] | +- external:dependency4:jar:2.4:compile",
+ "[INFO]",
+ "[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m2 ---",
+ "[INFO] internal:m2:jar:1.2",
+ "[INFO] +- internal:m1:jar:1.1:compile",
+ "[INFO] | +- external:dependency4:jar:2.4:compile");
+ }
+
+ @Test
+ void testTreeParsing() {
+ final Map<String, Set<Dependency>> dependenciesByModule =
+ DependencyParser.parseDependencyTreeOutput(getTestDependencyTree());
+
+ assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2");
+ assertThat(dependenciesByModule.get("m1"))
+ .containsExactlyInAnyOrder(
+ Dependency.create("external", "dependency1", "2.1", "compile", false),
+ Dependency.create("external", "dependency2", "2.2", "compile", true),
+ Dependency.create("external", "dependency3", "2.3", "provided", true),
+ Dependency.create("external", "dependency4", "2.4", "compile", false));
+ assertThat(dependenciesByModule.get("m2"))
+ .containsExactlyInAnyOrder(
+ Dependency.create("internal", "m1", "1.1", "compile", false),
+ Dependency.create("external", "dependency4", "2.4", "compile", false));
+ }
+
+ @Test
+ void testTreeLineParsingGroupId() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:compile"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.getGroupId()).isEqualTo("external"));
+ }
+
+ @Test
+ void testTreeLineParsingArtifactId() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:compile"))
+ .hasValueSatisfying(
+ dependency ->
+ assertThat(dependency.getArtifactId()).isEqualTo("dependency1"));
+ }
+
+ @Test
+ void testTreeLineParsingVersion() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:compile"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0"));
+ }
+
+ @Test
+ void testTreeLineParsingScope() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:provided"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.getScope()).hasValue("provided"));
+ }
+
+ @Test
+ void testTreeLineParsingWithNonJarType() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:pom:1.0:compile"))
+ .hasValue(Dependency.create("external", "dependency1", "1.0", "compile", false));
+ }
+
+ @Test
+ void testTreeLineParsingWithClassifier() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:some_classifier:1.0:compile"))
+ .hasValue(Dependency.create("external", "dependency1", "1.0", "compile", false));
+ }
+
+ @Test
+ void testTreeLineParsingWithoutOptional() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:compile"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.isOptional()).hasValue(false));
+ }
+
+ @Test
+ void testTreeLineParsingWithOptional() {
+ assertThat(
+ DependencyParser.parseTreeDependency(
+ "[INFO] +- external:dependency1:jar:1.0:compile (optional)"))
+ .hasValueSatisfying(
+ dependency -> assertThat(dependency.isOptional()).hasValue(true));
+ }
+}