You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/11/02 15:51:55 UTC
[2/3] flink git commit: [FLINK-7778] [build] Shade ZooKeeper
dependency (part 2)
[FLINK-7778] [build] Shade ZooKeeper dependency (part 2)
This closes #4927
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d368a07a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d368a07a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d368a07a
Branch: refs/heads/master
Commit: d368a07a2146c45a7f994cab0bb55524432a843d
Parents: 4d02823
Author: zentol <ch...@apache.org>
Authored: Tue Oct 31 11:26:48 2017 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Nov 2 16:51:25 2017 +0100
----------------------------------------------------------------------
.../store/ZooKeeperMesosWorkerStore.java | 11 ++-
flink-runtime/pom.xml | 27 +++---
.../runtime/zookeeper/ZookeeperAccess.java | 66 +++++++++++++++
flink-shaded-curator-recipes/pom.xml | 88 ++++++++++++++++++++
flink-shaded-hadoop/pom.xml | 5 --
pom.xml | 1 +
tools/travis_mvn_watchdog.sh | 16 ++++
7 files changed, 188 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
index 92e4416..738f99e 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
@@ -25,17 +25,16 @@ import org.apache.flink.runtime.zookeeper.ZooKeeperSharedCount;
import org.apache.flink.runtime.zookeeper.ZooKeeperSharedValue;
import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore;
import org.apache.flink.runtime.zookeeper.ZooKeeperVersionedValue;
+import org.apache.flink.runtime.zookeeper.ZookeeperAccess;
import org.apache.flink.util.FlinkException;
import org.apache.mesos.Protos;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.ConcurrentModificationException;
import java.util.List;
import scala.Option;
@@ -218,15 +217,15 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
try {
workersInZooKeeper.addAndLock(path, worker);
LOG.debug("Added {} in ZooKeeper.", worker);
- } catch (KeeperException.NodeExistsException ex) {
- throw new ConcurrentModificationException("ZooKeeper unexpectedly modified", ex);
+ } catch (Exception ex) {
+ throw ZookeeperAccess.wrapIfZooKeeperNodeExistsException(ex);
}
} else {
try {
workersInZooKeeper.replace(path, currentVersion, worker);
LOG.debug("Updated {} in ZooKeeper.", worker);
- } catch (KeeperException.NoNodeException ex) {
- throw new ConcurrentModificationException("ZooKeeper unexpectedly modified", ex);
+ } catch (Exception ex) {
+ throw ZookeeperAccess.wrapIfZooKeeperNoNodeException(ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 46990a9..481de6d 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -178,17 +178,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <!-- curator shades guava, but still has a dependency on it. -->
- <!-- We can safely exclude Guava here -->
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-curator-recipes</artifactId>
+ <version>${project.version}</version>
</dependency>
<!-- test dependencies -->
@@ -448,7 +440,6 @@ under the License.
<include>com.typesafe.akka:akka-remote_*</include>
<include>io.netty:netty</include>
<include>org.uncommons.maths:uncommons-maths</include>
- <include>org.apache.curator:*</include>
<include>org.apache.zookeeper:*</include>
</includes>
</artifactSet>
@@ -463,16 +454,22 @@ under the License.
</relocation>
<relocation>
<pattern>org.apache.curator</pattern>
- <shadedPattern>org.apache.flink.shaded.org.apache.curator</shadedPattern>
+ <shadedPattern>org.apache.flink.shaded.curator.org.apache.curator</shadedPattern>
+ <excludes>
+ <!-- Do not relocate curator-test. This leads to problems for downstream
+ users of runtime test classes that make use of it as the relocated
+ dependency is not included in the test-jar.-->
+ <exclude>org.apache.curator.test.*</exclude>
+ </excludes>
</relocation>
<relocation>
<pattern>org.apache.zookeeper</pattern>
- <shadedPattern>org.apache.flink.shaded.org.apache.zookeeper</shadedPattern>
+ <shadedPattern>org.apache.flink.shaded.zookeeper.org.apache.zookeeper</shadedPattern>
</relocation>
<!-- jute is already shaded into the ZooKeeper jar -->
<relocation>
<pattern>org.apache.jute</pattern>
- <shadedPattern>org.apache.flink.shaded.org.apache.zookeeper.jute</shadedPattern>
+ <shadedPattern>org.apache.flink.shaded.zookeeper.org.apache.zookeeper.jute</shadedPattern>
</relocation>
</relocations>
<filters>
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZookeeperAccess.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZookeeperAccess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZookeeperAccess.java
new file mode 100644
index 0000000..2c6160a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZookeeperAccess.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.zookeeper;
+
+import org.apache.zookeeper.KeeperException;
+
+import java.util.ConcurrentModificationException;
+
+/**
+ * Utility class providing access to relocated zookeeper classes.
+ *
+ * <p>This class is necessary as flink-runtime relocates its ZooKeeper dependency.
+ * Other modules may still depend on this dependency but will encounter a ClassNotFoundException
+ * on access as they don't apply the relocation pattern of flink-runtime.
+ */
+public final class ZookeeperAccess {
+
+ private ZookeeperAccess(){
+ }
+
+ /**
+ * Wraps and returns the given exception in a {@link ConcurrentModificationException} if it is a
+ * {@link org.apache.zookeeper.KeeperException.NodeExistsException}. Otherwise the
+ * given exception is returned.
+ *
+ * @param ex exception to wrap
+ * @return wrapping ConcurrentModificationException if it is a NodeExistsException, otherwise the given exception
+ */
+ public static Exception wrapIfZooKeeperNodeExistsException(Exception ex) {
+ if (ex instanceof KeeperException.NodeExistsException) {
+ return new ConcurrentModificationException("ZooKeeper unexpectedly modified", ex);
+ }
+ return ex;
+ }
+
+ /**
+ * Wraps and returns the given exception in a {@link ConcurrentModificationException} if it is a
+ * {@link org.apache.zookeeper.KeeperException.NoNodeException}. Otherwise the
+ * given exception is returned.
+ *
+ * @param ex exception to wrap
+ * @return wrapping ConcurrentModificationException if it is a NoNodeException, otherwise the given exception
+ */
+ public static Exception wrapIfZooKeeperNoNodeException(Exception ex) {
+ if (ex instanceof KeeperException.NoNodeException) {
+ return new ConcurrentModificationException("ZooKeeper unexpectedly modified", ex);
+ }
+ return ex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/flink-shaded-curator-recipes/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-curator-recipes/pom.xml b/flink-shaded-curator-recipes/pom.xml
new file mode 100644
index 0000000..06d3821
--- /dev/null
+++ b/flink-shaded-curator-recipes/pom.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>1.4-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-shaded-curator-recipes</artifactId>
+ <name>flink-shaded-curator-recipes</name>
+
+ <packaging>jar</packaging>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.curator:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>com.google.guava:guava</artifact>
+ <!-- Shade guava classes that are not included by curator -->
+ <includes>
+ <include>com/google/common/base/Function.class</include>
+ <include>com/google/common/base/Predicate.class</include>
+ <include>com/google/common/reflect/TypeToken.class</include>
+ </includes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.flink.curator.shaded.com.google.common</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/flink-shaded-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index f4b0816..ba90fc9 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -53,11 +53,6 @@ under the License.
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a5f99f7..faddab4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,7 @@ under the License.
<module>tools/force-shading</module>
<module>flink-annotations</module>
<module>flink-shaded-hadoop</module>
+ <module>flink-shaded-curator-recipes</module>
<module>flink-core</module>
<module>flink-java</module>
<module>flink-java8</module>
http://git-wip-us.apache.org/repos/asf/flink/blob/d368a07a/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index b7e3709..978bc9f 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -325,6 +325,22 @@ check_shaded_artifacts() {
return 1
fi
+ ZOOKEEPER=`cat allClasses | grep '^org/apache/zookeeper' | wc -l`
+ if [ "$ZOOKEEPER" != "0" ]; then
+ echo "=============================================================================="
+ echo "Detected '$ZOOKEEPER' unshaded org.apache.zookeeper classes in fat jar"
+ echo "=============================================================================="
+ return 1
+ fi
+
+ CURATOR=`cat allClasses | grep '^org/apache/curator' | wc -l`
+ if [ "$CURATOR" != "0" ]; then
+ echo "=============================================================================="
+ echo "Detected '$CURATOR' unshaded org.apache.curator classes in fat jar"
+ echo "=============================================================================="
+ return 1
+ fi
+
FLINK_PYTHON=`cat allClasses | grep '^org/apache/flink/python' | wc -l`
if [ "$FLINK_PYTHON" != "0" ]; then
echo "=============================================================================="