[flink] 01/02: [hotfix][tests] Refactor MapR FS Tests

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 0033448f0bc92104e134a9dc7c53be9da936315d
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Tue Jul 30 11:04:23 2019 +0200

    [hotfix][tests] Refactor MapR FS Tests
---
 .../runtime/fs/maprfs/FileSystemAccessTest.java    |  42 -------
 .../flink/runtime/fs/maprfs/MapRFreeTests.java     |  74 ------------
 .../flink/runtime/fs/maprfs/MapRFsFactoryTest.java |  86 +++-----------
 .../runtime/fs/maprfs/MapRNotInClassPathTest.java  | 129 +++++++++++++++++++++
 4 files changed, 146 insertions(+), 185 deletions(-)

diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java
deleted file mode 100644
index aee50ba..0000000
--- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.fs.maprfs;
-
-import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.util.TestLogger;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * This test checks that the file system is properly accessible through the
- * service loading abstraction.
- */
-public class FileSystemAccessTest extends TestLogger {
-
-	@Test
-	public void testGetMapRFs() throws Exception {
-		final Path path = new Path("maprfs:///my/path");
-
-		FileSystem fs = path.getFileSystem();
-		assertEquals(path.toUri().getScheme(), fs.getUri().getScheme());
-	}
-}
diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java
deleted file mode 100644
index 110fce3..0000000
--- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.fs.maprfs;
-
-import org.apache.flink.configuration.Configuration;
-
-import java.io.IOException;
-import java.net.URI;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * A class with tests that require to be run in a MapR/Hadoop-free environment,
- * to test proper error handling when no Hadoop classes are available.
- *
- * <p>This class must be dynamically loaded in a MapR/Hadoop-free class loader.
- */
-// this class is only instantiated via reflection
-@SuppressWarnings("unused")
-public class MapRFreeTests {
-
-	public static void test() throws Exception {
-		// make sure no MapR or Hadoop FS classes are in the classpath
-		try {
-			Class.forName("com.mapr.fs.MapRFileSystem");
-			fail("Cannot run test when MapR classes are in the classpath");
-		}
-		catch (ClassNotFoundException ignored) {}
-
-		try {
-			Class.forName("org.apache.hadoop.fs.FileSystem");
-			fail("Cannot run test when Hadoop classes are in the classpath");
-		}
-		catch (ClassNotFoundException ignored) {}
-
-		try {
-			Class.forName("org.apache.hadoop.conf.Configuration");
-			fail("Cannot run test when Hadoop classes are in the classpath");
-		}
-		catch (ClassNotFoundException ignored) {}
-
-		// this method should complete without a linkage error
-		final MapRFsFactory factory = new MapRFsFactory();
-
-		// this method should also complete without a linkage error
-		factory.configure(new Configuration());
-
-		try {
-			factory.create(new URI("maprfs://somehost:9000/root/dir"));
-			fail("This statement should fail with an exception");
-		}
-		catch (IOException e) {
-			assertTrue(e.getMessage().contains("MapR"));
-			assertTrue(e.getMessage().contains("classpath"));
-		}
-	}
-}
diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java
index 781c3d7..2c8357a 100644
--- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java
+++ b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java
@@ -18,103 +18,51 @@
 
 package org.apache.flink.runtime.fs.maprfs;
 
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.util.ClassLoaderUtils;
-import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.core.fs.FileSystemKind;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.Test;
 
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URI;
-import java.net.URL;
-import java.net.URLClassLoader;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 /**
- * Tests for the MapRFsFactory.
+ * Tests for the {@link MapRFsFactory}.
  */
 public class MapRFsFactoryTest extends TestLogger {
 
-	/**
-	 * This test validates that the factory can be instantiated and configured even
-	 * when MapR and Hadoop classes are missing from the classpath.
-	 */
 	@Test
-	public void testInstantiationWithoutMapRClasses() throws Exception {
-		// we do reflection magic here to instantiate the test in another class
-		// loader, to make sure no MapR and Hadoop classes are in the classpath
+	public void testMapRFsScheme() throws Exception {
+		final Path path = new Path("maprfs:///my/path");
 
-		final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRFreeTests";
+		final FileSystem fs = path.getFileSystem();
 
-		final URL[] urls = ClassLoaderUtils.getClasspathURLs();
+		assertEquals(path.toUri().getScheme(), fs.getUri().getScheme());
+	}
 
-		ClassLoader parent = getClass().getClassLoader();
-		ClassLoader maprFreeClassLoader = new MapRFreeClassLoader(urls, parent);
-		Class<?> testClass = Class.forName(testClassName, false, maprFreeClassLoader);
-		Method m = testClass.getDeclaredMethod("test");
+	@Test
+	public void testMapRFsKind() throws Exception {
+		final Path path = new Path("maprfs:///my/path");
 
-		try {
-			m.invoke(null);
-		}
-		catch (InvocationTargetException e) {
-			ExceptionUtils.rethrowException(e.getTargetException(), "exception in method");
-		}
+		final FileSystem fs = path.getFileSystem();
+
+		assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
 	}
 
 	@Test
-	public void testCreateFsWithAuthority() throws Exception {
-		final URI uri = URI.create("maprfs://localhost:12345/");
-
-		MapRFsFactory factory = new MapRFsFactory();
+	public void testCreateWithAuthorityNoCldbFails() throws Exception {
+		final Path path = new Path("maprfs://localhost:12345/");
 
 		try {
-			factory.create(uri);
+			path.getFileSystem();
 			fail("should have failed with an exception");
 		}
 		catch (IOException e) {
 			// expected, because we have no CLDB config available
 		}
 	}
-
-	@Test
-	public void testCreateFsWithMissingAuthority() throws Exception {
-		final URI uri = URI.create("maprfs:///my/path");
-
-		MapRFsFactory factory = new MapRFsFactory();
-		factory.configure(new Configuration());
-
-		FileSystem fs = factory.create(uri);
-		assertEquals("maprfs", fs.getUri().getScheme());
-	}
-
-	// ------------------------------------------------------------------------
-
-	private static final class MapRFreeClassLoader extends URLClassLoader {
-
-		private final ClassLoader properParent;
-
-		MapRFreeClassLoader(URL[] urls, ClassLoader parent) {
-			super(urls, null);
-			properParent = parent;
-		}
-
-		@Override
-		public Class<?> loadClass(String name) throws ClassNotFoundException {
-			if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) {
-				throw new ClassNotFoundException(name);
-			}
-			else if (name.startsWith("org.apache.log4j")) {
-				return properParent.loadClass(name);
-			}
-			else {
-				return super.loadClass(name);
-			}
-		}
-	}
 }
diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java
new file mode 100644
index 0000000..2d274ee
--- /dev/null
+++ b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.fs.maprfs;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.ClassLoaderUtils;
+import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.function.RunnableWithException;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URLClassLoader;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * A class with tests that require to be run in a MapR/Hadoop-free environment,
+ * to test proper error handling when no Hadoop classes are available.
+ *
+ * <p>This class must be dynamically loaded in a MapR/Hadoop-free class loader.
+ */
+public class MapRNotInClassPathTest extends TestLogger {
+
+	@Test
+	public void testInstantiationWhenMapRClassesAreMissing() throws Exception {
+		final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRNotInClassPathTest$TestRunner";
+		final ClassLoader cl = new MapRFreeClassLoader(getClass().getClassLoader());
+
+		final RunnableWithException testRunner = Class
+			.forName(testClassName, false, cl)
+			.asSubclass(RunnableWithException.class)
+			.newInstance();
+
+		testRunner.run();
+	}
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * The tests that need to run in a special classloader.
+	 */
+	@SuppressWarnings("unused")
+	public static final class TestRunner implements RunnableWithException {
+
+		@Override
+		public void run() throws Exception {
+			// make sure no MapR or Hadoop FS classes are in the classpath
+			try {
+				Class.forName("com.mapr.fs.MapRFileSystem");
+				fail("Cannot run test when MapR classes are in the classpath");
+			}
+			catch (ClassNotFoundException ignored) {}
+
+			try {
+				Class.forName("org.apache.hadoop.fs.FileSystem");
+				fail("Cannot run test when Hadoop classes are in the classpath");
+			}
+			catch (ClassNotFoundException ignored) {}
+
+			try {
+				Class.forName("org.apache.hadoop.conf.Configuration");
+				fail("Cannot run test when Hadoop classes are in the classpath");
+			}
+			catch (ClassNotFoundException ignored) {}
+
+			// this method should complete without a linkage error
+			final MapRFsFactory factory = new MapRFsFactory();
+
+			// this method should also complete without a linkage error
+			factory.configure(new Configuration());
+
+			try {
+				factory.create(new URI("maprfs://somehost:9000/root/dir"));
+				fail("This statement should fail with an exception");
+			}
+			catch (IOException e) {
+				assertTrue(e.getMessage().contains("MapR"));
+				assertTrue(e.getMessage().contains("classpath"));
+			}
+		}
+	}
+
+	// ------------------------------------------------------------------------
+
+	/**
+	 * A special classloader that filters "org.apache.hadoop.*" and "com.mapr.*" classes.
+	 */
+	private static final class MapRFreeClassLoader extends URLClassLoader {
+
+		private final ClassLoader properParent;
+
+		MapRFreeClassLoader(ClassLoader parent) {
+			super(ClassLoaderUtils.getClasspathURLs(), null);
+			properParent = parent;
+		}
+
+		@Override
+		public Class<?> loadClass(String name) throws ClassNotFoundException {
+			if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) {
+				throw new ClassNotFoundException(name);
+			}
+			else if (name.equals(RunnableWithException.class.getName()) || name.startsWith("org.apache.log4j")) {
+				return properParent.loadClass(name);
+			}
+			else {
+				return super.loadClass(name);
+			}
+		}
+	}
+}