You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/05/10 20:39:41 UTC
[1/8] flink git commit: [FLINK-6508] [build] Include licenses of
bundled/shaded dependencies where required
Repository: flink
Updated Branches:
refs/heads/release-1.3 81f58bae1 -> 062163344
[FLINK-6508] [build] Include licenses of bundled/shaded dependencies where required
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1b217371
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1b217371
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1b217371
Branch: refs/heads/release-1.3
Commit: 1b2173712dea5a5b95633af82623bd45785965ce
Parents: 6d7ffcc
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 16:42:15 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
flink-core/packaged_licenses/LICENSE.asm.txt | 31 +++++++++++++++++++
flink-java/packaged_licenses/LICENSE.asm.txt | 31 +++++++++++++++++++
.../packaged_licenses/LICENSE.janino.txt | 31 +++++++++++++++++++
.../packaged_licenses/LICENSE.reflections.txt | 14 +++++++++
flink-runtime/packaged_licenses/LICENSE.asm.txt | 31 +++++++++++++++++++
flink-scala/packaged_licenses/LICENSE.asm.txt | 31 +++++++++++++++++++
pom.xml | 32 +++++++++++++++++++-
7 files changed, 200 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-core/packaged_licenses/LICENSE.asm.txt
----------------------------------------------------------------------
diff --git a/flink-core/packaged_licenses/LICENSE.asm.txt b/flink-core/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000..62ffbcc
--- /dev/null
+++ b/flink-core/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-java/packaged_licenses/LICENSE.asm.txt
----------------------------------------------------------------------
diff --git a/flink-java/packaged_licenses/LICENSE.asm.txt b/flink-java/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000..62ffbcc
--- /dev/null
+++ b/flink-java/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt b/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
new file mode 100644
index 0000000..ef871e2
--- /dev/null
+++ b/flink-libraries/flink-table/packaged_licenses/LICENSE.janino.txt
@@ -0,0 +1,31 @@
+Janino - An embedded Java[TM] compiler
+
+Copyright (c) 2001-2016, Arno Unkrig
+Copyright (c) 2015-2016 TIBCO Software Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials
+ provided with the distribution.
+ 3. Neither the name of JANINO nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt b/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
new file mode 100644
index 0000000..236d111
--- /dev/null
+++ b/flink-libraries/flink-table/packaged_licenses/LICENSE.reflections.txt
@@ -0,0 +1,14 @@
+ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sa...@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-runtime/packaged_licenses/LICENSE.asm.txt
----------------------------------------------------------------------
diff --git a/flink-runtime/packaged_licenses/LICENSE.asm.txt b/flink-runtime/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000..62ffbcc
--- /dev/null
+++ b/flink-runtime/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/flink-scala/packaged_licenses/LICENSE.asm.txt
----------------------------------------------------------------------
diff --git a/flink-scala/packaged_licenses/LICENSE.asm.txt b/flink-scala/packaged_licenses/LICENSE.asm.txt
new file mode 100644
index 0000000..62ffbcc
--- /dev/null
+++ b/flink-scala/packaged_licenses/LICENSE.asm.txt
@@ -0,0 +1,31 @@
+ASM: a very small and fast Java bytecode manipulation framework
+
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/1b217371/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1c978c6..85e3474 100644
--- a/pom.xml
+++ b/pom.xml
@@ -912,6 +912,9 @@ under the License.
<exclude>**/resources/**/bootstrap*</exclude>
<exclude>flink-clients/src/main/resources/web-docs/js/*d3.js</exclude>
+ <!-- the licenses that are re-bundled -->
+ <exclude>**/packaged_licenses/LICENSE.*.txt</exclude>
+
<!-- web dashboard config JSON files -->
<exclude>flink-runtime-web/web-dashboard/package.json</exclude>
<exclude>flink-runtime-web/web-dashboard/bower.json</exclude>
@@ -1115,6 +1118,34 @@ under the License.
</executions>
</plugin>
+ <!-- make sure all licenses that need to be repackaged are in the target folder -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ </configuration>
+ <executions>
+ <execution>
+ <id>copy-packaged-licenses</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes/META-INF/license/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/packaged_licenses</directory>
+ <includes>
+ <include>LICENSE.*.txt</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<!-- We use shading in all packages for relocating some classes, such as
Guava and ASM.
By doing so, users adding Flink as a dependency won't run into conflicts.
@@ -1185,7 +1216,6 @@ under the License.
</execution>
</executions>
</plugin>
-
</plugins>
<!-- Plugin configurations for plugins activated in sub-projects -->
[4/8] flink git commit: [FLINK-6515] [runtime] Fix classloading of
JavaSerializer
Posted by se...@apache.org.
[FLINK-6515] [runtime] Fix classloading of JavaSerializer
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cc3512ee
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cc3512ee
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cc3512ee
Branch: refs/heads/release-1.3
Commit: cc3512ee9bbcc965278b43642cc9481f77027c4f
Parents: 95fd2d3
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 11:20:07 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
flink-runtime/pom.xml | 10 +-
.../flink/runtime/state/JavaSerializer.java | 17 ++-
.../flink/runtime/state/JavaSerializerTest.java | 116 +++++++++++++++++
.../flink/core/testutils/CommonTestUtils.java | 124 +++++++++++++++++++
4 files changed, 257 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/cc3512ee/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index a6b9513..2b42cc5 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -162,7 +162,7 @@ under the License.
<version>${project.version}</version>
</dependency>
- <!-- core dependencies -->
+ <!-- test dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
@@ -172,6 +172,14 @@ under the License.
</dependency>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/cc3512ee/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
index d49b1d2..5252b3d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/JavaSerializer.java
@@ -24,12 +24,12 @@ import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.InstantiationUtil;
import java.io.IOException;
import java.io.Serializable;
-@SuppressWarnings("serial")
@Internal
final class JavaSerializer<T extends Serializable> extends TypeSerializerSingleton<T> {
@@ -47,11 +47,10 @@ final class JavaSerializer<T extends Serializable> extends TypeSerializerSinglet
@Override
public T copy(T from) {
-
try {
- return InstantiationUtil.clone(from);
+ return InstantiationUtil.clone(from, Thread.currentThread().getContextClassLoader());
} catch (IOException | ClassNotFoundException e) {
- throw new RuntimeException("Could not copy instance of " + from + '.', e);
+ throw new FlinkRuntimeException("Could not copy element via serialization: " + from, e);
}
}
@@ -62,7 +61,7 @@ final class JavaSerializer<T extends Serializable> extends TypeSerializerSinglet
@Override
public int getLength() {
- return 0;
+ return -1;
}
@Override
@@ -74,7 +73,8 @@ final class JavaSerializer<T extends Serializable> extends TypeSerializerSinglet
public T deserialize(DataInputView source) throws IOException {
try {
return InstantiationUtil.deserializeObject(
- new DataInputViewStream(source), Thread.currentThread().getContextClassLoader());
+ new DataInputViewStream(source),
+ Thread.currentThread().getContextClassLoader());
} catch (ClassNotFoundException e) {
throw new IOException("Could not deserialize object.", e);
}
@@ -87,9 +87,8 @@ final class JavaSerializer<T extends Serializable> extends TypeSerializerSinglet
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
- int size = source.readInt();
- target.writeInt(size);
- target.write(source, size);
+ T tmp = deserialize(source);
+ serialize(tmp, target);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/cc3512ee/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
new file mode 100644
index 0000000..de6fbce
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/JavaSerializerTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.api.common.typeutils.SerializerTestBase;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.Serializable;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.*;
+
+/**
+ * A test that verifies that the {@link JavaSerializer} properly handles class loading.
+ */
+public class JavaSerializerTest extends SerializerTestBase<Serializable> {
+
+ /** Class loader for the object that is not in the test class path */
+ private static final ClassLoader CLASS_LOADER =
+ new URLClassLoader(new URL[0], JavaSerializerTest.class.getClassLoader());
+
+ /** An object that is not in the test class path */
+ private static final Serializable OBJECT_OUT_OF_CLASSPATH =
+ CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
+
+ // ------------------------------------------------------------------------
+
+ private ClassLoader originalClassLoader;
+
+ @Before
+ public void setupClassLoader() {
+ originalClassLoader = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+ }
+
+ @After
+ public void restoreOriginalClassLoader() {
+ Thread.currentThread().setContextClassLoader(originalClassLoader);
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void guardTest() {
+ // make sure that this test's assumptions hold
+ try {
+ Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
+ fail("Test ineffective: The test class that should not be on the classpath is actually on the classpath.");
+ } catch (ClassNotFoundException e) {
+ // expected
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ protected TypeSerializer<Serializable> createSerializer() {
+ Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+ return new JavaSerializer<>();
+ }
+
+ @Override
+ protected int getLength() {
+ return -1;
+ }
+
+ @Override
+ protected Class<Serializable> getTypeClass() {
+ return Serializable.class;
+ }
+
+ @Override
+ protected Serializable[] getTestData() {
+ return new Serializable[] {
+ new Integer(42),
+ new File("/some/path/that/I/made/up"),
+
+ // an object that is not in the classpath
+ OBJECT_OUT_OF_CLASSPATH,
+
+ // an object that is in the classpath with a nested object not in the classpath
+ new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
+ };
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void testInstantiate() {
+ // this serializer does not support instantiation
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/cc3512ee/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
index 639b065..cf2bb7f 100644
--- a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/CommonTestUtils.java
@@ -29,7 +29,12 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.Serializable;
import java.lang.reflect.Field;
+import java.security.CodeSource;
+import java.security.Permissions;
+import java.security.ProtectionDomain;
+import java.security.cert.Certificate;
import java.util.Map;
import static org.junit.Assert.fail;
@@ -178,4 +183,123 @@ public class CommonTestUtils {
throw new RuntimeException(e1);
}
}
+
+ // ------------------------------------------------------------------------
+ // Testing of objects not in the application class loader
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new class that is not part of the classpath that the current JVM uses, and
+ * instantiates it.
+ *
+ * <p>This method uses {@link #createClassNotInClassPath(ClassLoader)} to define the new class.
+ *
+ * @param targetClassLoader The class loader to attach the class to
+ * @return The object instantiated from the newly defined class.
+ */
+ public static Serializable createObjectForClassNotInClassPath(ClassLoader targetClassLoader) {
+ try {
+ Class<? extends Serializable> clazz = createClassNotInClassPath(targetClassLoader);
+ return clazz.newInstance();
+ }
+ catch (Exception e) {
+ throw new AssertionError("test setup broken", e);
+ }
+ }
+
+ /**
+ * Creates a new class that is not part of the classpath that the current JVM uses.
+ * The class is ad-hoc defined and attached to the given ClassLoader.
+ *
+ * @param targetClassLoader The class loader to attach the class to
+ * @return The newly defined class
+ */
+ public static Class<? extends Serializable> createClassNotInClassPath(ClassLoader targetClassLoader) {
+ final byte[] classData = {-54, -2, -70, -66, 0, 0, 0, 51, 0, 65, 10, 0, 15, 0, 43, 7, 0, 44,
+ 10, 0, 2, 0, 43, 10, 0, 2, 0, 45, 9, 0, 7, 0, 46, 10, 0, 15, 0, 47, 7, 0, 48, 7, 0,
+ 49, 10, 0, 8, 0, 43, 8, 0, 50, 10, 0, 8, 0, 51, 10, 0, 8, 0, 52, 10, 0, 8, 0, 53, 10,
+ 0, 8, 0, 54, 7, 0, 55, 7, 0, 56, 1, 0, 16, 115, 101, 114, 105, 97, 108, 86, 101, 114,
+ 115, 105, 111, 110, 85, 73, 68, 1, 0, 1, 74, 1, 0, 13, 67, 111, 110, 115, 116, 97, 110,
+ 116, 86, 97, 108, 117, 101, 5, -1, -1, -1, -1, -1, -1, -1, -3, 1, 0, 6, 114, 97, 110,
+ 100, 111, 109, 1, 0, 6, 60, 105, 110, 105, 116, 62, 1, 0, 3, 40, 41, 86, 1, 0, 4, 67,
+ 111, 100, 101, 1, 0, 15, 76, 105, 110, 101, 78, 117, 109, 98, 101, 114, 84, 97, 98, 108,
+ 101, 1, 0, 18, 76, 111, 99, 97, 108, 86, 97, 114, 105, 97, 98, 108, 101, 84, 97, 98,
+ 108, 101, 1, 0, 4, 116, 104, 105, 115, 1, 0, 35, 76, 111, 114, 103, 47, 97, 112, 97, 99,
+ 104, 101, 47, 102, 108, 105, 110, 107, 47, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108,
+ 105, 122, 97, 98, 108, 101, 59, 1, 0, 6, 101, 113, 117, 97, 108, 115, 1, 0, 21, 40, 76,
+ 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 79, 98, 106, 101, 99, 116, 59, 41, 90, 1, 0,
+ 1, 111, 1, 0, 18, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 79, 98, 106, 101, 99,
+ 116, 59, 1, 0, 4, 116, 104, 97, 116, 1, 0, 13, 83, 116, 97, 99, 107, 77, 97, 112, 84, 97,
+ 98, 108, 101, 7, 0, 48, 1, 0, 8, 104, 97, 115, 104, 67, 111, 100, 101, 1, 0, 3, 40, 41,
+ 73, 1, 0, 8, 116, 111, 83, 116, 114, 105, 110, 103, 1, 0, 20, 40, 41, 76, 106, 97, 118, 97,
+ 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 59, 1, 0, 10, 83, 111, 117, 114,
+ 99, 101, 70, 105, 108, 101, 1, 0, 21, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105,
+ 122, 97, 98, 108, 101, 46, 106, 97, 118, 97, 12, 0, 23, 0, 24, 1, 0, 16, 106, 97, 118, 97,
+ 47, 117, 116, 105, 108, 47, 82, 97, 110, 100, 111, 109, 12, 0, 57, 0, 58, 12, 0, 22, 0, 18,
+ 12, 0, 59, 0, 60, 1, 0, 33, 111, 114, 103, 47, 97, 112, 97, 99, 104, 101, 47, 102, 108, 105,
+ 110, 107, 47, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105, 122, 97, 98, 108, 101, 1,
+ 0, 23, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 66, 117,
+ 105, 108, 100, 101, 114, 1, 0, 24, 84, 101, 115, 116, 83, 101, 114, 105, 97, 108, 105, 122,
+ 97, 98, 108, 101, 123, 114, 97, 110, 100, 111, 109, 61, 12, 0, 61, 0, 62, 12, 0, 61, 0, 63,
+ 12, 0, 61, 0, 64, 12, 0, 39, 0, 40, 1, 0, 16, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47,
+ 79, 98, 106, 101, 99, 116, 1, 0, 20, 106, 97, 118, 97, 47, 105, 111, 47, 83, 101, 114, 105,
+ 97, 108, 105, 122, 97, 98, 108, 101, 1, 0, 8, 110, 101, 120, 116, 76, 111, 110, 103, 1, 0,
+ 3, 40, 41, 74, 1, 0, 8, 103, 101, 116, 67, 108, 97, 115, 115, 1, 0, 19, 40, 41, 76, 106, 97,
+ 118, 97, 47, 108, 97, 110, 103, 47, 67, 108, 97, 115, 115, 59, 1, 0, 6, 97, 112, 112, 101,
+ 110, 100, 1, 0, 45, 40, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105,
+ 110, 103, 59, 41, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110,
+ 103, 66, 117, 105, 108, 100, 101, 114, 59, 1, 0, 28, 40, 74, 41, 76, 106, 97, 118, 97, 47,
+ 108, 97, 110, 103, 47, 83, 116, 114, 105, 110, 103, 66, 117, 105, 108, 100, 101, 114, 59, 1,
+ 0, 28, 40, 67, 41, 76, 106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 83, 116, 114, 105, 110,
+ 103, 66, 117, 105, 108, 100, 101, 114, 59, 0, 33, 0, 7, 0, 15, 0, 1, 0, 16, 0, 2, 0, 26, 0,
+ 17, 0, 18, 0, 1, 0, 19, 0, 0, 0, 2, 0, 20, 0, 18, 0, 22, 0, 18, 0, 0, 0, 4, 0, 1, 0, 23, 0,
+ 24, 0, 1, 0, 25, 0, 0, 0, 69, 0, 3, 0, 1, 0, 0, 0, 19, 42, -73, 0, 1, 42, -69, 0, 2, 89, -73,
+ 0, 3, -74, 0, 4, -75, 0, 5, -79, 0, 0, 0, 2, 0, 26, 0, 0, 0, 14, 0, 3, 0, 0, 0, 30, 0, 4, 0,
+ 31, 0, 18, 0, 32, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 19, 0, 28, 0, 29, 0, 0, 0, 1, 0, 30, 0,
+ 31, 0, 1, 0, 25, 0, 0, 0, -116, 0, 4, 0, 3, 0, 0, 0, 47, 42, 43, -90, 0, 5, 4, -84, 43, -58,
+ 0, 14, 42, -74, 0, 6, 43, -74, 0, 6, -91, 0, 5, 3, -84, 43, -64, 0, 7, 77, 42, -76, 0, 5, 44,
+ -76, 0, 5, -108, -102, 0, 7, 4, -89, 0, 4, 3, -84, 0, 0, 0, 3, 0, 26, 0, 0, 0, 18, 0, 4, 0, 0,
+ 0, 36, 0, 7, 0, 37, 0, 24, 0, 39, 0, 29, 0, 40, 0, 27, 0, 0, 0, 32, 0, 3, 0, 0, 0, 47, 0, 28,
+ 0, 29, 0, 0, 0, 0, 0, 47, 0, 32, 0, 33, 0, 1, 0, 29, 0, 18, 0, 34, 0, 29, 0, 2, 0, 35, 0, 0,
+ 0, 13, 0, 5, 7, 14, 1, -4, 0, 20, 7, 0, 36, 64, 1, 0, 1, 0, 37, 0, 38, 0, 1, 0, 25, 0, 0, 0,
+ 56, 0, 5, 0, 1, 0, 0, 0, 14, 42, -76, 0, 5, 42, -76, 0, 5, 16, 32, 125, -125, -120, -84, 0, 0,
+ 0, 2, 0, 26, 0, 0, 0, 6, 0, 1, 0, 0, 0, 46, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 14, 0, 28, 0,
+ 29, 0, 0, 0, 1, 0, 39, 0, 40, 0, 1, 0, 25, 0, 0, 0, 70, 0, 3, 0, 1, 0, 0, 0, 28, -69, 0, 8,
+ 89, -73, 0, 9, 18, 10, -74, 0, 11, 42, -76, 0, 5, -74, 0, 12, 16, 125, -74, 0, 13, -74, 0, 14,
+ -80, 0, 0, 0, 2, 0, 26, 0, 0, 0, 6, 0, 1, 0, 0, 0, 51, 0, 27, 0, 0, 0, 12, 0, 1, 0, 0, 0, 28,
+ 0, 28, 0, 29, 0, 0, 0, 1, 0, 41, 0, 0, 0, 2, 0, 42,};
+
+ try {
+ // define a class into the classloader
+ Class<?> clazz = getUnsafe().defineClass(
+ "org.apache.flink.TestSerializable",
+ classData, 0, classData.length,
+ targetClassLoader,
+ new ProtectionDomain(new CodeSource(null, (Certificate[]) null), new Permissions()));
+
+ return clazz.asSubclass(Serializable.class);
+ }
+ catch (Exception e) {
+ throw new AssertionError("test setup broken", e);
+ }
+ }
+
+ @SuppressWarnings("restriction")
+ private static sun.misc.Unsafe getUnsafe() {
+ try {
+ Field unsafeField = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ return (sun.misc.Unsafe) unsafeField.get(null);
+ } catch (SecurityException e) {
+ throw new RuntimeException("Could not access the sun.misc.Unsafe handle, permission denied by security manager.", e);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException("The static handle field in sun.misc.Unsafe was not found.");
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException("Bug: Illegal argument reflection access for static field.", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Access to sun.misc.Unsafe is forbidden by the runtime.", e);
+ } catch (Throwable t) {
+ throw new RuntimeException("Unclassified error while trying to access the sun.misc.Unsafe handle.", t);
+ }
+ }
}
[3/8] flink git commit: [FLINK-6501] [build] Add NOTICE transformers
to shading
Posted by se...@apache.org.
[FLINK-6501] [build] Add NOTICE transformers to shading
This makes sure that transitive NOTICE files are added to the shaded JAR files.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/95fd2d37
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/95fd2d37
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/95fd2d37
Branch: refs/heads/release-1.3
Commit: 95fd2d371eae43be1ea2534efe653af1a7095d0b
Parents: 9b8ace9
Author: Stephan Ewen <se...@apache.org>
Authored: Mon May 8 20:58:26 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
pom.xml | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/95fd2d37/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 71ad779..1c978c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1173,6 +1173,14 @@ under the License.
<shadedPattern>org.apache.flink.shaded.org.objectweb.asm</shadedPattern>
</relocation>
</relocations>
+ <transformers>
+ <!-- The service transformer is needed to merge META-INF/services files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ <!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ </transformers>
</configuration>
</execution>
</executions>
[8/8] flink git commit: [hotfix] [tests] Share proper test mini
cluster for tests in DistributedCacheTest
Posted by se...@apache.org.
[hotfix] [tests] Share proper test mini cluster for tests in DistributedCacheTest
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/06216334
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/06216334
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/06216334
Branch: refs/heads/release-1.3
Commit: 062163344f8785733503837b83a4d063baa3c4cf
Parents: 1b21737
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 20:57:08 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:53 2017 +0200
----------------------------------------------------------------------
.../distributedCache/DistributedCacheTest.java | 56 ++++++++++++++++----
1 file changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/06216334/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
index 19bcf76..21aa40a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/distributedCache/DistributedCacheTest.java
@@ -22,19 +22,30 @@ import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
+import org.apache.flink.test.util.TestEnvironment;
import org.apache.flink.util.Collector;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.assertTrue;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
-import java.io.*;
+import static org.junit.Assert.assertTrue;
-import java.util.*;
+public class DistributedCacheTest extends AbstractTestBase {
-public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
public static final String data
= "machen\n"
+ "zeit\n"
@@ -42,6 +53,31 @@ public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
+ "keiner\n"
+ "meine\n";
+ private static final int PARALLELISM = 4;
+
+ private static LocalFlinkMiniCluster cluster;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ cluster = TestBaseUtils.startCluster(1, PARALLELISM, false, false, true);
+ TestStreamEnvironment.setAsContext(cluster, PARALLELISM);
+ TestEnvironment.setAsContext(cluster, PARALLELISM);
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ TestStreamEnvironment.unsetAsContext();
+ TestEnvironment.unsetAsContext();
+ TestBaseUtils.stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
+ }
+
+ // ------------------------------------------------------------------------
+
+ public DistributedCacheTest() {
+ super(new Configuration());
+ }
+
+ // ------------------------------------------------------------------------
@Test
public void testStreamingDistributedCache() throws Exception {
@@ -68,12 +104,12 @@ public class DistributedCacheTest extends StreamingMultipleProgramsTestBase {
@Override
public void open(Configuration conf) throws IOException {
File file = getRuntimeContext().getDistributedCache().getFile("cache_test");
- BufferedReader reader = new BufferedReader(new FileReader(file));
- String tempString;
- while ((tempString = reader.readLine()) != null) {
- wordList.add(tempString);
+ try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
+ String tempString;
+ while ((tempString= reader.readLine()) != null) {
+ wordList.add(tempString);
+ }
}
- reader.close();
}
@Override
[6/8] flink git commit: [hotfix] [build] Move JSON dependency in
flink-storm into proper section
Posted by se...@apache.org.
[hotfix] [build] Move JSON dependency in flink-storm into proper section
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9b8ace94
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9b8ace94
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9b8ace94
Branch: refs/heads/release-1.3
Commit: 9b8ace9417a6144aa3047c4ccf0c12e009bb78a3
Parents: 81f58ba
Author: Stephan Ewen <se...@apache.org>
Authored: Mon May 8 20:58:01 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
flink-contrib/flink-storm/pom.xml | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9b8ace94/flink-contrib/flink-storm/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml
index b6cc61c..cc300b3 100644
--- a/flink-contrib/flink-storm/pom.xml
+++ b/flink-contrib/flink-storm/pom.xml
@@ -164,6 +164,12 @@ under the License.
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1</version>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
@@ -173,11 +179,6 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
- <version>1.1</version>
- </dependency>
</dependencies>
[7/8] flink git commit: [hotfix] [core] Minor code cleanups in
JavaSerializer and SerializerTestBase
Posted by se...@apache.org.
[hotfix] [core] Minor code cleanups in JavaSerializer and SerializerTestBase
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2696cf34
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2696cf34
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2696cf34
Branch: refs/heads/release-1.3
Commit: 2696cf34411a466154d88b7fa31b92bf568c64c9
Parents: cc3512e
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 11:28:55 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
.../java/typeutils/runtime/kryo/JavaSerializer.java | 4 ++--
.../api/common/typeutils/SerializerTestBase.java | 16 ++++++++++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2696cf34/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
index a51647c..711c814 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/JavaSerializer.java
@@ -45,7 +45,7 @@ public class JavaSerializer<T> extends Serializer<T> {
public JavaSerializer() {}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({"unchecked", "rawtypes"})
@Override
public void write(Kryo kryo, Output output, T o) {
try {
@@ -62,7 +62,7 @@ public class JavaSerializer<T> extends Serializer<T> {
}
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({"unchecked", "rawtypes"})
@Override
public T read(Kryo kryo, Input input, Class aClass) {
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/2696cf34/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index a846703..f2879ac 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -55,7 +55,13 @@ import org.junit.Test;
public abstract class SerializerTestBase<T> extends TestLogger {
protected abstract TypeSerializer<T> createSerializer();
-
+
+ /**
+ * Gets the expected length for the serializer's {@link TypeSerializer#getLength()} method.
+ *
+ * <p>The expected length should be positive, for fix-length data types, or {@code -1} for
+ * variable-length types.
+ */
protected abstract int getLength();
protected abstract Class<T> getTypeClass();
@@ -124,9 +130,15 @@ public abstract class SerializerTestBase<T> extends TestLogger {
@Test
public void testGetLength() {
+ final int len = getLength();
+
+ if (len == 0) {
+ fail("Broken serializer test base - zero length cannot be the expected length");
+ }
+
try {
TypeSerializer<T> serializer = getSerializer();
- assertEquals(getLength(), serializer.getLength());
+ assertEquals(len, serializer.getLength());
}
catch (Exception e) {
System.err.println(e.getMessage());
[5/8] flink git commit: [hotfix] [tests] Add a lightweight test for
classloading in the Kryo Serializer
Posted by se...@apache.org.
[hotfix] [tests] Add a lightweight test for classloading in the Kryo Serializer
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/65fcca0a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/65fcca0a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/65fcca0a
Branch: refs/heads/release-1.3
Commit: 65fcca0a43cf32427f8fd13d197815d1c5b7d078
Parents: 2696cf3
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 12:37:28 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
.../kryo/KryoSerializerClassLoadingTest.java | 113 +++++++++++++++++++
1 file changed, 113 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/65fcca0a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
new file mode 100644
index 0000000..9823e11
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerClassLoadingTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.SerializerTestBase;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.core.testutils.CommonTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.fail;
+
+/**
+ * This test validates that the Kryo-based serializer handles classes with custom
+ * class loaders correctly.
+ */
+public class KryoSerializerClassLoadingTest extends SerializerTestBase<Object> {
+
+ /** Class loader for the object that is not in the test class path */
+ private static final ClassLoader CLASS_LOADER =
+ new URLClassLoader(new URL[0], KryoSerializerClassLoadingTest.class.getClassLoader());
+
+ /** An object that is not in the test class path */
+ private static final Serializable OBJECT_OUT_OF_CLASSPATH =
+ CommonTestUtils.createObjectForClassNotInClassPath(CLASS_LOADER);
+
+ // ------------------------------------------------------------------------
+
+ private ClassLoader originalClassLoader;
+
+ @Before
+ public void setupClassLoader() {
+ originalClassLoader = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(CLASS_LOADER);
+ }
+
+ @After
+ public void restoreOriginalClassLoader() {
+ Thread.currentThread().setContextClassLoader(originalClassLoader);
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void guardTestAssumptions() {
+ try {
+ Class.forName(OBJECT_OUT_OF_CLASSPATH.getClass().getName());
+ fail("This test's assumptions are broken");
+ }
+ catch (ClassNotFoundException ignored) {
+ // expected
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ protected TypeSerializer<Object> createSerializer() {
+ return new KryoSerializer<>(Object.class, new ExecutionConfig());
+ }
+
+ @Override
+ protected int getLength() {
+ return -1;
+ }
+
+ @Override
+ protected Class<Object> getTypeClass() {
+ return Object.class;
+ }
+
+ @Override
+ protected Object[] getTestData() {
+ return new Object[] {
+ new Integer(7),
+
+ // an object whose class is not on the classpath
+ OBJECT_OUT_OF_CLASSPATH,
+
+ // an object whose class IS on the classpath with a nested object whose class
+ // is NOT on the classpath
+ new Tuple1<>(OBJECT_OUT_OF_CLASSPATH)
+ };
+ }
+
+ @Override
+ public void testInstantiate() {
+ // this serializer does not support instantiation
+ }
+}
[2/8] flink git commit: [hotfix] [gelly] Support log output when
running examples in the IDE
Posted by se...@apache.org.
[hotfix] [gelly] Support log output when running examples in the IDE
This adds the dependencies / config files necessary for log output
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6d7ffcc0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6d7ffcc0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6d7ffcc0
Branch: refs/heads/release-1.3
Commit: 6d7ffcc09ad9195ec05b887fb644e72fbcd2ca60
Parents: 65fcca0a
Author: Stephan Ewen <se...@apache.org>
Authored: Wed May 10 16:39:53 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed May 10 21:08:37 2017 +0200
----------------------------------------------------------------------
flink-libraries/flink-gelly-examples/pom.xml | 15 ++++++++++
.../src/main/resources/log4j.properties | 23 ++++++++++++++++
.../src/main/resources/logback.xml | 29 ++++++++++++++++++++
3 files changed, 67 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6d7ffcc0/flink-libraries/flink-gelly-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
index e95aa37..cd8d807 100644
--- a/flink-libraries/flink-gelly-examples/pom.xml
+++ b/flink-libraries/flink-gelly-examples/pom.xml
@@ -75,6 +75,21 @@
<scope>compile</scope>
</dependency>
+ <!-- Add a logging Framework, to make the examples produce -->
+ <!-- logs when executing in the IDE -->
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/6d7ffcc0/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties b/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
new file mode 100644
index 0000000..da32ea0
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/resources/log4j.properties
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
http://git-wip-us.apache.org/repos/asf/flink/blob/6d7ffcc0/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml b/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml
new file mode 100644
index 0000000..95f2d04
--- /dev/null
+++ b/flink-libraries/flink-gelly-examples/src/main/resources/logback.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file