You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2019/01/26 15:29:25 UTC
[flink] 01/03: [FLINK-11316][tests] Drop JarFileCreator
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 6c01cf92b6050b66e5bf1ea9057266c9edaedff4
Author: tison <wa...@gmail.com>
AuthorDate: Mon Jan 14 23:47:33 2019 +0800
[FLINK-11316][tests] Drop JarFileCreator
---
.../apache/flink/runtime/util/JarFileCreator.java | 222 -----------------
.../flink/runtime/util/ClassLoaderUtilsTest.java | 32 ++-
.../flink/runtime/util/JarFileCreatorTest.java | 274 ---------------------
3 files changed, 29 insertions(+), 499 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/JarFileCreator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/JarFileCreator.java
deleted file mode 100644
index d77c9f8..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/JarFileCreator.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.util;
-
-import org.apache.flink.shaded.asm5.org.objectweb.asm.ClassReader;
-import org.apache.flink.shaded.asm5.org.objectweb.asm.Opcodes;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashSet;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-
-/**
- * This is an auxiliary program which creates a jar file from a set of classes.
- * <p>
- * This class is thread-safe.
- *
- */
-public class JarFileCreator {
-
- /**
- * The file extension of java classes.
- */
- private static final String CLASS_EXTENSION = ".class";
-
- /**
- * A set of classes which shall be included in the final jar file.
- */
- private final Set<Class<?>> classSet = new HashSet<Class<?>>();
-
- /**
- * The final jar file.
- */
- private final File outputFile;
-
- /**
- * The namespace of the dependencies to be packaged.
- */
- private final Set<String> packages = new HashSet<String>();
-
- /**
- * Constructs a new jar file creator.
- *
- * @param outputFile
- * the file which shall contain the output data, i.e. the final jar file
- */
- public JarFileCreator(final File outputFile) {
-
- this.outputFile = outputFile;
- }
-
- /**
- * Adds a {@link Class} object to the set of classes which shall eventually be included in the jar file.
- *
- * @param clazz
- * the class to be added to the jar file.
- */
- public synchronized JarFileCreator addClass(final Class<?> clazz) {
-
- this.classSet.add(clazz);
- String name = clazz.getName();
- int n = name.lastIndexOf('.');
- if (n > -1) {
- name = name.substring(0, n);
- }
- return addPackage(name);
- }
-
- /**
- * Manually specify the package of the dependencies.
- *
- * @param p
- * the package to be included.
- */
- public synchronized JarFileCreator addPackage(String p) {
- this.packages.add(p);
- return this;
- }
-
- /**
- * Manually specify the packages of the dependencies.
- *
- * @param packages
- * the packages to be included.
- */
- public synchronized JarFileCreator addPackages(String[] packages) {
- for (String p : packages) {
- addPackage(p);
- }
- return this;
- }
-
- /**
- * Add the dependencies within the given packages automatically.
- * @throws IOException
- * throw if an error occurs while read the class file.
- */
- private synchronized void addDependencies() throws IOException {
- List<String> dependencies = new ArrayList<String>();
- for (Class clazz : classSet) {
- dependencies.add(clazz.getName());
- }
- //Traverse the dependency tree using BFS.
- int head = 0;
- while (head != dependencies.size()) {
- DependencyVisitor v = new DependencyVisitor(Opcodes.ASM5);
- v.addNameSpace(this.packages);
- InputStream classInputStream = null;
- String name = dependencies.get(head);
- try {
- Class clazz = Class.forName(name);
- int n = name.lastIndexOf('.');
- String className = null;
- if (n > -1) {
- className = name.substring(n + 1, name.length());
- }
- classInputStream = clazz.getResourceAsStream(className + CLASS_EXTENSION);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e.getMessage());
- }
- new ClassReader(classInputStream).accept(v, 0);
- classInputStream.close();
-
- //Update the BFS queue.
- Set<String> classPackages = v.getPackages();
- for (String s : classPackages) {
- if (!dependencies.contains(s.replace('/','.'))) {
- dependencies.add(s.replace('/','.'));
- }
- }
- head++;
- }
-
- for (String dependency : dependencies) {
- try {
- this.classSet.add(Class.forName(dependency));
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e.getMessage());
- }
- }
- }
-
- /**
- * Creates a jar file which contains the previously added class. The content of the jar file is written to
- * <code>outputFile</code> which has been provided to the constructor. If <code>outputFile</code> already exists, it
- * is overwritten by this operation.
- *
- * @throws IOException
- * thrown if an error occurs while writing to the output file
- */
- public synchronized void createJarFile() throws IOException {
- //Retrieve dependencies automatically
- addDependencies();
-
- // Temporary buffer for the stream copy
- final byte[] buf = new byte[128];
-
- // Check if output file is valid
- if (this.outputFile == null) {
- throw new IOException("Output file is null");
- }
-
- // If output file already exists, delete it
- if (this.outputFile.exists()) {
- this.outputFile.delete();
- }
-
- try ( FileOutputStream fos = new FileOutputStream(this.outputFile); JarOutputStream jos = new JarOutputStream(fos, new Manifest())) {
- final Iterator<Class<?>> it = this.classSet.iterator();
- while (it.hasNext()) {
-
- final Class<?> clazz = it.next();
- final String entry = clazz.getName().replace('.', '/') + CLASS_EXTENSION;
-
- jos.putNextEntry(new JarEntry(entry));
-
- String name = clazz.getName();
- int n = name.lastIndexOf('.');
- String className = null;
- if (n > -1) {
- className = name.substring(n + 1, name.length());
- }
- //Using the part after last dot instead of class.getSimpleName() could resolve the problem of inner class.
- final InputStream classInputStream = clazz.getResourceAsStream(className + CLASS_EXTENSION);
-
- int num = classInputStream.read(buf);
- while (num != -1) {
- jos.write(buf, 0, num);
- num = classInputStream.read(buf);
- }
-
- classInputStream.close();
- jos.closeEntry();
- }
- }
- }
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/ClassLoaderUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/ClassLoaderUtilsTest.java
index abd0590..4fa37c0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/ClassLoaderUtilsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/ClassLoaderUtilsTest.java
@@ -24,9 +24,13 @@ import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.InputStream;
import java.net.URL;
import java.net.URLClassLoader;
+import java.util.jar.JarEntry;
import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
/**
* Tests that validate the {@link ClassLoaderUtil}.
@@ -41,9 +45,7 @@ public class ClassLoaderUtilsTest {
try {
// file with jar contents
validJar = File.createTempFile("flink-url-test", ".tmp");
- JarFileCreator jarFileCreator = new JarFileCreator(validJar);
- jarFileCreator.addClass(ClassLoaderUtilsTest.class);
- jarFileCreator.createJarFile();
+ createValidJar(validJar);
// validate that the JAR is correct and the test setup is not broken
JarFile jarFile = null;
@@ -106,6 +108,30 @@ public class ClassLoaderUtilsTest {
}
}
}
+
+ private void createValidJar(File validJar) throws Exception {
+ final Class<?> clazz = ClassLoaderUtilsTest.class;
+ final String classExtension = ".class";
+ final byte[] buf = new byte[128];
+
+ try (FileOutputStream fos = new FileOutputStream(validJar); JarOutputStream jos = new JarOutputStream(fos, new Manifest())) {
+ String entry = clazz.getName().replace('.', '/') + classExtension;
+ jos.putNextEntry(new JarEntry(entry));
+
+ String name = clazz.getName();
+ int n = name.lastIndexOf('.');
+ String className = (n > -1) ? name.substring(n + 1) : name;
+
+ final InputStream classInputStream = clazz.getResourceAsStream(className + classExtension);
+
+ for (int num = classInputStream.read(buf); num != -1; num = classInputStream.read(buf)) {
+ jos.write(buf, 0, num);
+ }
+
+ classInputStream.close();
+ jos.closeEntry();
+ }
+ }
@Test
public void testWithAppClassLoader() {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/JarFileCreatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/JarFileCreatorTest.java
deleted file mode 100644
index 60e3292..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/JarFileCreatorTest.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.util;
-
-import org.apache.flink.runtime.util.jartestprogram.FilterWithIndirection;
-import org.apache.flink.runtime.util.jartestprogram.FilterWithLambda;
-import org.apache.flink.runtime.util.jartestprogram.FilterWithMethodReference;
-import org.apache.flink.runtime.util.jartestprogram.WordCountWithAnonymousClass;
-import org.apache.flink.runtime.util.jartestprogram.WordCountWithExternalClass;
-import org.apache.flink.runtime.util.jartestprogram.WordCountWithExternalClass2;
-import org.apache.flink.runtime.util.jartestprogram.WordCountWithInnerClass;
-import org.apache.flink.runtime.util.jartestprogram.AnonymousInStaticMethod;
-import org.apache.flink.runtime.util.jartestprogram.AnonymousInNonStaticMethod;
-import org.apache.flink.runtime.util.jartestprogram.AnonymousInNonStaticMethod2;
-import org.apache.flink.runtime.util.jartestprogram.NestedAnonymousInnerClass;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.jar.JarInputStream;
-import java.util.zip.ZipEntry;
-
-public class JarFileCreatorTest {
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- //anonymous inner class in static method accessing a local variable in its closure.
- @Test
- public void TestAnonymousInnerClassTrick1() throws Exception {
- File out = tempFolder.newFile("jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(AnonymousInStaticMethod.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInStaticMethod$1.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInStaticMethod$A.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInStaticMethod.class");
-
- Assert.assertTrue("Jar file for Anonymous Inner Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- //anonymous inner class in non static method accessing a local variable in its closure.
- @Test
- public void TestAnonymousInnerClassTrick2() throws Exception {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(AnonymousInNonStaticMethod.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod$1.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod$A.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod.class");
-
- Assert.assertTrue("Jar file for Anonymous Inner Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- //anonymous inner class in non static method accessing a field of its enclosing class.
- @Test
- public void TestAnonymousInnerClassTrick3() throws Exception {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(AnonymousInNonStaticMethod2.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod2$1.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod2$A.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/AnonymousInNonStaticMethod2.class");
-
- Assert.assertTrue("Jar file for Anonymous Inner Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- //anonymous inner class in an anonymous inner class accessing a field of the outermost enclosing class.
- @Test
- public void TestAnonymousInnerClassTrick4() throws Exception {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(NestedAnonymousInnerClass.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/NestedAnonymousInnerClass.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/NestedAnonymousInnerClass$1$1.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/NestedAnonymousInnerClass$1.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/NestedAnonymousInnerClass$A.class");
-
- Assert.assertTrue("Jar file for Anonymous Inner Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- @Ignore // this is currently not supported (see FLINK-9520)
- @Test
- public void testFilterWithMethodReference() throws Exception {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(FilterWithMethodReference.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/FilterWithMethodReference.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordFilter.class");
-
- Assert.assertTrue("Jar file for Java 8 method reference is not correct", validate(ans, out));
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void testFilterWithLambda() throws Exception{
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(FilterWithLambda.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/FilterWithLambda.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordFilter.class");
-
- Assert.assertTrue("Jar file for Java 8 lambda is not correct", validate(ans, out));
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void testFilterWithIndirection() throws Exception {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(FilterWithIndirection.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/FilterWithIndirection.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordFilter.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/UtilFunctionWrapper$UtilFunction.class");
-
- Assert.assertTrue("Jar file for java 8 lambda is not correct", validate(ans, out));
- Assert.assertTrue(out.delete());
- }
-
- //----------------------------------------------------------------------------------------------
- //Word Count Example
-
- @Test
- public void TestExternalClass() throws IOException {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(WordCountWithExternalClass.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/StaticData.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithExternalClass.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/ExternalTokenizer.class");
-
- Assert.assertTrue("Jar file for External Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void TestInnerClass() throws IOException {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(WordCountWithInnerClass.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/StaticData.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithInnerClass.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithInnerClass$Tokenizer.class");
-
- Assert.assertTrue("Jar file for Inner Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void TestAnonymousClass() throws IOException {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(WordCountWithAnonymousClass.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/StaticData.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithAnonymousClass.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithAnonymousClass$1.class");
-
- Assert.assertTrue("Jar file for Anonymous Class is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void TestExtendIdentifier() throws IOException {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(WordCountWithExternalClass2.class)
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/StaticData.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithExternalClass2.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/ExternalTokenizer2.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/ExternalTokenizer.class");
-
- Assert.assertTrue("Jar file for Extend Identifier is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- @Test
- public void TestUDFPackage() throws IOException {
- File out = new File(tempFolder.getRoot(), "jarcreatortest.jar");
- JarFileCreator jfc = new JarFileCreator(out);
- jfc.addClass(WordCountWithInnerClass.class)
- .addPackage("org.apache.flink.util")
- .createJarFile();
-
- Set<String> ans = new HashSet<>();
- ans.add("org/apache/flink/runtime/util/jartestprogram/StaticData.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithInnerClass.class");
- ans.add("org/apache/flink/runtime/util/jartestprogram/WordCountWithInnerClass$Tokenizer.class");
- ans.add("org/apache/flink/util/Collector.class");
-
- Assert.assertTrue("Jar file for UDF package is not correct", validate(ans, out));
-
- Assert.assertTrue(out.delete());
- }
-
- private boolean validate(Set<String> expected, File out) throws IOException {
- int count = expected.size();
- try (JarInputStream jis = new JarInputStream(new FileInputStream(out))) {
- ZipEntry ze;
- while ((ze = jis.getNextEntry()) != null) {
- count--;
- expected.remove(ze.getName());
- }
- }
- return count == 0 && expected.size() == 0;
- }
-}
-