You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2022/09/14 21:07:36 UTC
[flink] 03/04: [FLINK-29301] Automatically determine modules skipping deployment
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7ed8022da8e782d82a33b230bd6ae204f1526ea3
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Sep 14 13:35:45 2022 +0200
[FLINK-29301] Automatically determine modules skipping deployment
In order to work with external modules we can't rely on a static list for modules that skip deployment (== where the NOTICE contents are irrelevant). We can determine that easily from the build output.
---
.../tools/ci/licensecheck/NoticeFileChecker.java | 63 +++++++++++++++++-----
.../modules-skipping-deployment.modulelist | 43 ---------------
2 files changed, 50 insertions(+), 56 deletions(-)
diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
index 8ea2cec2945..8ba7f95c449 100644
--- a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
+++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java
@@ -37,6 +37,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -51,9 +52,6 @@ public class NoticeFileChecker {
private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class);
- private static final List<String> MODULES_SKIPPING_DEPLOYMENT =
- loadFromResources("modules-skipping-deployment.modulelist");
-
private static final List<String> MODULES_DEFINING_EXCESS_DEPENDENCIES =
loadFromResources("modules-defining-excess-dependencies.modulelist");
@@ -64,6 +62,19 @@ public class NoticeFileChecker {
private static final Pattern SHADE_INCLUDE_MODULE_PATTERN =
Pattern.compile(".*Including ([^:]+):([^:]+):jar:([^ ]+) in the shaded jar");
+ // Examples:
+ //
+ // Deployment on CI with alternative repo
+ // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---
+ // [INFO] Using alternate deployment repository.../tmp/flink-validation-deployment
+ //
+ // Skipped deployment:
+ // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---
+ // [INFO] Skipping artifact deployment
+ private static final Pattern DEPLOY_MODULE_PATTERN =
+ Pattern.compile(
+ ".maven-deploy-plugin:.*:deploy .* @ (?<module>[^ _]+)(_[0-9.]+)? --.*");
+
// Examples:
// "- org.apache.htrace:htrace-core:3.1.0-incubating"
// or
@@ -80,13 +91,22 @@ public class NoticeFileChecker {
parseModulesFromBuildResult(buildResult),
DependencyParser.parseDependencyCopyOutput(buildResult.toPath()));
+ final Set<String> deployedModules = parseDeployedModulesFromBuildResult(buildResult);
+
LOG.info(
"Extracted "
+ + deployedModules.size()
+ + " modules that were deployed of which "
+ modulesWithBundledDependencies.keySet().size()
- + " modules with a total of "
+ + " bundle dependencies with a total of "
+ modulesWithBundledDependencies.values().size()
+ " dependencies");
+ final HashSet<String> moduleSkippingDeployment =
+ new HashSet<>(modulesWithBundledDependencies.keySet());
+ moduleSkippingDeployment.removeAll(deployedModules);
+ moduleSkippingDeployment.forEach(modulesWithBundledDependencies::removeAll);
+
// find modules producing a shaded-jar
List<Path> noticeFiles = findNoticeFiles(root);
LOG.info("Found {} NOTICE files to check", noticeFiles.size());
@@ -127,15 +147,13 @@ public class NoticeFileChecker {
.map(NoticeFileChecker::getModuleFromNoticeFile)
.collect(Collectors.toList()));
for (String moduleWithoutNotice : shadingModules) {
- if (!MODULES_SKIPPING_DEPLOYMENT.contains(moduleWithoutNotice)) {
- LOG.error(
- "Module {} is missing a NOTICE file. It has shaded dependencies: {}",
- moduleWithoutNotice,
- modulesWithShadedDependencies.get(moduleWithoutNotice).stream()
- .map(Dependency::toString)
- .collect(Collectors.joining("\n\t", "\n\t", "")));
- severeIssueCount++;
- }
+ LOG.error(
+ "Module {} is missing a NOTICE file. It has shaded dependencies: {}",
+ moduleWithoutNotice,
+ modulesWithShadedDependencies.get(moduleWithoutNotice).stream()
+ .map(Dependency::toString)
+ .collect(Collectors.joining("\n\t", "\n\t", "")));
+ severeIssueCount++;
}
return severeIssueCount;
}
@@ -317,6 +335,25 @@ public class NoticeFileChecker {
return result;
}
+ private static Set<String> parseDeployedModulesFromBuildResult(File buildResult)
+ throws IOException {
+ final Set<String> deployedModules = new HashSet<>();
+ try (Stream<String> linesStream = Files.lines(buildResult.toPath())) {
+ final Iterator<String> lines = linesStream.iterator();
+ while (lines.hasNext()) {
+ final String line = lines.next();
+ final Matcher matcher = DEPLOY_MODULE_PATTERN.matcher(line);
+ if (matcher.find()) {
+ final String module = matcher.group("module");
+ if (lines.hasNext() && !lines.next().contains("Skipping artifact deployment")) {
+ deployedModules.add(module);
+ }
+ }
+ }
+ }
+ return deployedModules;
+ }
+
private static List<String> loadFromResources(String fileName) {
try {
try (BufferedReader bufferedReader =
diff --git a/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist b/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist
deleted file mode 100644
index 7b0be584fee..00000000000
--- a/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist
+++ /dev/null
@@ -1,43 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# These modules are not deployed to maven central, despite their use of the shade plugin.
-
-flink-examples-streaming-gcp-pubsub
-flink-yarn-tests
-flink-docs
-flink-datastream-allround-test
-flink-queryable-state-test
-flink-confluent-schema-registry
-flink-stream-stateful-job-upgrade-test
-flink-elasticsearch7-test
-flink-stream-state-ttl-test
-flink-state-evolution-test
-flink-elasticsearch6-test
-flink-rocksdb-state-memory-control-test
-flink-python-test
-flink-streaming-kinesis-test
-flink-tpch-test
-flink-streaming-kafka-test-base
-flink-heavy-deployment-stress-test
-flink-high-parallelism-iterations-test
-flink-end-to-end-tests-common-kafka
-flink-end-to-end-tests-pulsar
-flink-end-to-end-tests-elasticsearch7
-flink-end-to-end-tests-elasticsearch6
-flink-sql-gateway-test