You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/05/01 21:23:57 UTC
[1/2] beam git commit: [BEAM-1676] Internalize ClassPath from Guava
21.0 to fix the failures related to having whitespaces in resource names.
Repository: beam
Updated Branches:
refs/heads/master 3ae944130 -> c2a555783
[BEAM-1676] Internalize ClassPath from Guava 21.0 to fix the failures related to having whitespaces in resource names.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0788d3db
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0788d3db
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0788d3db
Branch: refs/heads/master
Commit: 0788d3db82f7de3f749a279274208186ad928f3d
Parents: 3ae9441
Author: Stas Levin <st...@apache.org>
Authored: Sun Apr 30 13:56:40 2017 +0300
Committer: Dan Halperin <dh...@google.com>
Committed: Mon May 1 14:23:45 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/beam/findbugs-filter.xml | 6 +
.../org/apache/beam/sdk/util/ApiSurface.java | 4 +-
.../org/apache/beam/sdk/util/ClassPath.java | 544 +++++++++++++++++++
3 files changed, 551 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
----------------------------------------------------------------------
diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
index d03fbf3..d1d8b4d 100644
--- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml
@@ -379,6 +379,12 @@
<!--[BEAM-421] Class doesn't override equals in superclass-->
</Match>
<Match>
+ <Class name="org.apache.beam.sdk.util.ClassPath$ClassInfo"/>
+ <Method name="equals"/>
+ <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS"/>
+ <!--[BEAM-1676] Class doesn't override equals in superclass-->
+ </Match>
+ <Match>
<Class name="org.apache.beam.sdk.util.AutoValue_GcsUtil_StorageObjectOrIOException"/>
<Bug pattern="NM_CLASS_NOT_EXCEPTION"/>
<!-- It is clear from the name that this class holds either StorageObject or IOException. -->
http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
index 9530e88..8c2b988 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java
@@ -33,8 +33,6 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
-import com.google.common.reflect.ClassPath;
-import com.google.common.reflect.ClassPath.ClassInfo;
import com.google.common.reflect.Invokable;
import com.google.common.reflect.Parameter;
import com.google.common.reflect.TypeToken;
@@ -353,7 +351,7 @@ public class ApiSurface {
ClassPath classPath = ClassPath.from(classLoader);
Set<Class<?>> newRootClasses = Sets.newHashSet();
- for (ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) {
+ for (ClassPath.ClassInfo classInfo : classPath.getTopLevelClassesRecursive(packageName)) {
Class clazz = classInfo.load();
if (exposed(clazz.getModifiers())) {
newRootClasses.add(clazz);
http://git-wip-us.apache.org/repos/asf/beam/blob/0788d3db/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
new file mode 100644
index 0000000..35632ed
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ClassPath.java
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.annotations.Beta;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Predicate;
+import com.google.common.base.Splitter;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.MultimapBuilder;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteSource;
+import com.google.common.io.CharSource;
+import com.google.common.io.Resources;
+import com.google.common.reflect.Reflection;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.Charset;
+import java.util.Enumeration;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Scans the source of a {@link ClassLoader} and finds all loadable classes and resources.
+ *
+ * <p><b>Warning:</b> Currently only {@link URLClassLoader} and only {@code file://} urls are
+ * supported.
+ * </p>
+ *
+ * <p>Based on Ben Yu's implementation in
+ * <a href="https://github.com/google/guava/blob/896c51abd32e136621c13d56b6130d0a72f4957a/guava/src/com/google/common/reflect/ClassPath.java">Guava</a>.
+ * </p>
+ *
+ * <p><b>Note:</b> Internalised here to avoid a forced upgrade to
+ * <a href="https://github.com/google/guava/releases/tag/v21.0">Guava 21.0 which requires
+ * Java 8.</a>
+ * </p>
+ */
+@Beta
+final class ClassPath {
+
+ private static final Logger logger = LoggerFactory.getLogger(ClassPath.class.getName());
+
+ private static final Predicate<ClassInfo> IS_TOP_LEVEL =
+ new Predicate<ClassInfo>() {
+
+ @Override
+ public boolean apply(ClassInfo info) {
+ return info != null && info.className.indexOf('$') == -1;
+ }
+ };
+
+ /** Separator for the Class-Path manifest attribute value in jar files. */
+ private static final Splitter CLASS_PATH_ATTRIBUTE_SEPARATOR =
+ Splitter.on(" ").omitEmptyStrings();
+
+ private static final String CLASS_FILE_NAME_EXTENSION = ".class";
+
+ private final ImmutableSet<ResourceInfo> resources;
+
+ private ClassPath(ImmutableSet<ResourceInfo> resources) {
+ this.resources = resources;
+ }
+
+ /**
+ * Returns a {@code ClassPath} representing all classes and resources loadable from {@code
+ * classloader} and its parent class loaders.
+ *
+ * <p><b>Warning:</b> Currently only {@link URLClassLoader} and only {@code file://} urls are
+ * supported.
+ *
+ * @throws IOException if the attempt to read class path resources (jar files or directories)
+ * failed.
+ */
+ public static ClassPath from(ClassLoader classloader) throws IOException {
+ DefaultScanner scanner = new DefaultScanner();
+ scanner.scan(classloader);
+ return new ClassPath(scanner.getResources());
+ }
+
+ /**
+ * Returns all resources loadable from the current class path, including the class files of all
+ * loadable classes but excluding the "META-INF/MANIFEST.MF" file.
+ */
+ public ImmutableSet<ResourceInfo> getResources() {
+ return resources;
+ }
+
+ /**
+ * Returns all classes loadable from the current class path.
+ *
+ * @since 16.0
+ */
+ public ImmutableSet<ClassInfo> getAllClasses() {
+ return FluentIterable.from(resources).filter(ClassInfo.class).toSet();
+ }
+
+ /** Returns all top level classes loadable from the current class path. */
+ public ImmutableSet<ClassInfo> getTopLevelClasses() {
+ return FluentIterable.from(resources).filter(ClassInfo.class).filter(IS_TOP_LEVEL).toSet();
+ }
+
+ /** Returns all top level classes whose package name is {@code packageName}. */
+ public ImmutableSet<ClassInfo> getTopLevelClasses(String packageName) {
+ checkNotNull(packageName);
+ ImmutableSet.Builder<ClassInfo> builder = ImmutableSet.builder();
+ for (ClassInfo classInfo : getTopLevelClasses()) {
+ if (classInfo.getPackageName().equals(packageName)) {
+ builder.add(classInfo);
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Returns all top level classes whose package name is {@code packageName} or starts with
+ * {@code packageName} followed by a '.'.
+ */
+ public ImmutableSet<ClassInfo> getTopLevelClassesRecursive(String packageName) {
+ checkNotNull(packageName);
+ String packagePrefix = packageName + '.';
+ ImmutableSet.Builder<ClassInfo> builder = ImmutableSet.builder();
+ for (ClassInfo classInfo : getTopLevelClasses()) {
+ if (classInfo.getName().startsWith(packagePrefix)) {
+ builder.add(classInfo);
+ }
+ }
+ return builder.build();
+ }
+
+ /**
+ * Represents a class path resource that can be either a class file or any other resource file
+ * loadable from the class path.
+ *
+ * @since 14.0
+ */
+ @Beta
+ public static class ResourceInfo {
+
+ private final String resourceName;
+
+ final ClassLoader loader;
+
+ static ResourceInfo of(String resourceName, ClassLoader loader) {
+ if (resourceName.endsWith(CLASS_FILE_NAME_EXTENSION)) {
+ return new ClassInfo(resourceName, loader);
+ } else {
+ return new ResourceInfo(resourceName, loader);
+ }
+ }
+
+ ResourceInfo(String resourceName, ClassLoader loader) {
+ this.resourceName = checkNotNull(resourceName);
+ this.loader = checkNotNull(loader);
+ }
+
+ /**
+ * Returns the url identifying the resource.
+ *
+ * <p>See {@link ClassLoader#getResource}
+ *
+ * @throws NoSuchElementException if the resource cannot be loaded through the class loader,
+ * despite physically existing in the class path.
+ */
+ public final URL url() {
+ URL url = loader.getResource(resourceName);
+ if (url == null) {
+ throw new NoSuchElementException(resourceName);
+ }
+ return url;
+ }
+
+ /**
+ * Returns a {@link ByteSource} view of the resource from which its bytes can be read.
+ *
+ * @throws NoSuchElementException if the resource cannot be loaded through the class loader,
+ * despite physically existing in the class path.
+ * @since 20.0
+ */
+ public final ByteSource asByteSource() {
+ return Resources.asByteSource(url());
+ }
+
+ /**
+ * Returns a {@link CharSource} view of the resource from which its bytes can be read as
+ * characters decoded with the given {@code charset}.
+ *
+ * @throws NoSuchElementException if the resource cannot be loaded through the class loader,
+ * despite physically existing in the class path.
+ * @since 20.0
+ */
+ public final CharSource asCharSource(Charset charset) {
+ return Resources.asCharSource(url(), charset);
+ }
+
+ /** Returns the fully qualified name of the resource. Such as "com/mycomp/foo/bar.txt". */
+ public final String getResourceName() {
+ return resourceName;
+ }
+
+ @Override
+ public int hashCode() {
+ return resourceName.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof ResourceInfo) {
+ ResourceInfo that = (ResourceInfo) obj;
+ return resourceName.equals(that.resourceName) && loader == that.loader;
+ }
+ return false;
+ }
+
+ // Do not change this arbitrarily. We rely on it for sorting ResourceInfo.
+ @Override
+ public String toString() {
+ return resourceName;
+ }
+ }
+
+ /**
+ * Represents a class that can be loaded through {@link #load}.
+ *
+ * @since 14.0
+ */
+ @Beta
+ static final class ClassInfo extends ResourceInfo {
+
+ private final String className;
+
+ ClassInfo(String resourceName, ClassLoader loader) {
+ super(resourceName, loader);
+ this.className = getClassName(resourceName);
+ }
+
+ /**
+ * Returns the package name of the class, without attempting to load the class.
+ *
+ * <p>Behaves identically to {@link Package#getName()} but does not require the class (or
+ * package) to be loaded.
+ */
+ public String getPackageName() {
+ return Reflection.getPackageName(className);
+ }
+
+ /**
+ * Returns the simple name of the underlying class as given in the source code.
+ *
+ * <p>Behaves identically to {@link Class#getSimpleName()} but does not require the class
+ * to be
+ * loaded.
+ */
+ public String getSimpleName() {
+ int lastDollarSign = className.lastIndexOf('$');
+ if (lastDollarSign != -1) {
+ String innerClassName = className.substring(lastDollarSign + 1);
+ // local and anonymous classes are prefixed with number (1,2,3...), anonymous classes are
+ // entirely numeric whereas local classes have the user supplied name as a suffix
+ return CharMatcher.digit().trimLeadingFrom(innerClassName);
+ }
+ String packageName = getPackageName();
+ if (packageName.isEmpty()) {
+ return className;
+ }
+
+ // Since this is a top level class, its simple name is always the part after package name.
+ return className.substring(packageName.length() + 1);
+ }
+
+ /**
+ * Returns the fully qualified name of the class.
+ *
+ * <p>Behaves identically to {@link Class#getName()} but does not require the class to be
+ * loaded.
+ */
+ public String getName() {
+ return className;
+ }
+
+ /**
+ * Loads (but doesn't link or initialize) the class.
+ *
+ * @throws LinkageError when there were errors in loading classes that this class depends on.
+ * For example, {@link NoClassDefFoundError}.
+ */
+ public Class<?> load() {
+ try {
+ return loader.loadClass(className);
+ } catch (ClassNotFoundException e) {
+ // Shouldn't happen, since the class name is read from the class path.
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return className;
+ }
+ }
+
+ /**
+ * Abstract class that scans through the class path represented by a {@link ClassLoader} and
+ * calls
+ * {@link #scanDirectory} and {@link #scanJarFile} for directories and jar files on the class
+ * path
+ * respectively.
+ */
+ abstract static class Scanner {
+
+ // We only scan each file once independent of the classloader that resource might be
+ // associated
+ // with.
+ private final Set<File> scannedUris = Sets.newHashSet();
+
+ public final void scan(ClassLoader classloader) throws IOException {
+ for (Map.Entry<File, ClassLoader> entry : getClassPathEntries(classloader).entrySet()) {
+ scan(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /** Called when a directory is scanned for resource files. */
+ protected abstract void scanDirectory(ClassLoader loader, File directory) throws IOException;
+
+ /** Called when a jar file is scanned for resource entries. */
+ protected abstract void scanJarFile(ClassLoader loader, JarFile file) throws IOException;
+
+ @VisibleForTesting
+ final void scan(File file, ClassLoader classloader) throws IOException {
+ if (scannedUris.add(file.getCanonicalFile())) {
+ scanFrom(file, classloader);
+ }
+ }
+
+ private void scanFrom(File file, ClassLoader classloader) throws IOException {
+ try {
+ if (!file.exists()) {
+ return;
+ }
+ } catch (SecurityException e) {
+ logger.warn("Cannot access " + file + ": " + e);
+ return;
+ }
+ if (file.isDirectory()) {
+ scanDirectory(classloader, file);
+ } else {
+ scanJar(file, classloader);
+ }
+ }
+
+ private void scanJar(File file, ClassLoader classloader) throws IOException {
+ JarFile jarFile;
+ try {
+ jarFile = new JarFile(file);
+ } catch (IOException e) {
+ // Not a jar file
+ return;
+ }
+ try {
+ for (File path : getClassPathFromManifest(file, jarFile.getManifest())) {
+ scan(path, classloader);
+ }
+ scanJarFile(classloader, jarFile);
+ } finally {
+ try {
+ jarFile.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+
+ /**
+ * Returns the class path URIs specified by the {@code Class-Path} manifest attribute,
+ * according
+ * to
+ * <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Main_Attributes">
+ * JAR File Specification</a>. If {@code manifest} is null, it means the jar file has no
+ * manifest, and an empty set will be returned.
+ */
+ @VisibleForTesting
+ static ImmutableSet<File> getClassPathFromManifest(File jarFile,
+ @Nullable Manifest manifest) {
+ if (manifest == null) {
+ return ImmutableSet.of();
+ }
+ ImmutableSet.Builder<File> builder = ImmutableSet.builder();
+ String classpathAttribute =
+ manifest.getMainAttributes().getValue(Attributes.Name.CLASS_PATH.toString());
+ if (classpathAttribute != null) {
+ for (String path : CLASS_PATH_ATTRIBUTE_SEPARATOR.split(classpathAttribute)) {
+ URL url;
+ try {
+ url = getClassPathEntry(jarFile, path);
+ } catch (MalformedURLException e) {
+ // Ignore bad entry
+ logger.warn("Invalid Class-Path entry: " + path);
+ continue;
+ }
+ if (url.getProtocol().equals("file")) {
+ builder.add(toFile(url));
+ }
+ }
+ }
+ return builder.build();
+ }
+
+ @VisibleForTesting
+ static ImmutableMap<File, ClassLoader> getClassPathEntries(ClassLoader classloader) {
+ LinkedHashMap<File, ClassLoader> entries = Maps.newLinkedHashMap();
+ // Search parent first, since it's the order ClassLoader#loadClass() uses.
+ ClassLoader parent = classloader.getParent();
+ if (parent != null) {
+ entries.putAll(getClassPathEntries(parent));
+ }
+ if (classloader instanceof URLClassLoader) {
+ URLClassLoader urlClassLoader = (URLClassLoader) classloader;
+ for (URL entry : urlClassLoader.getURLs()) {
+ if (entry.getProtocol().equals("file")) {
+ File file = toFile(entry);
+ if (!entries.containsKey(file)) {
+ entries.put(file, classloader);
+ }
+ }
+ }
+ }
+ return ImmutableMap.copyOf(entries);
+ }
+
+ /**
+ * Returns the absolute uri of the Class-Path entry value as specified in
+ * <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Main_Attributes">
+ * JAR File Specification</a>. Even though the specification only talks about relative urls,
+ * absolute urls are actually supported too (for example, in Maven surefire plugin).
+ */
+ @VisibleForTesting
+ static URL getClassPathEntry(File jarFile, String path) throws MalformedURLException {
+ return new URL(jarFile.toURI().toURL(), path);
+ }
+ }
+
+ @VisibleForTesting
+ static final class DefaultScanner extends Scanner {
+
+ private final SetMultimap<ClassLoader, String> resources =
+ MultimapBuilder.hashKeys().linkedHashSetValues().build();
+
+ ImmutableSet<ResourceInfo> getResources() {
+ ImmutableSet.Builder<ResourceInfo> builder = ImmutableSet.builder();
+ for (Map.Entry<ClassLoader, String> entry : resources.entries()) {
+ builder.add(ResourceInfo.of(entry.getValue(), entry.getKey()));
+ }
+ return builder.build();
+ }
+
+ @Override
+ protected void scanJarFile(ClassLoader classloader, JarFile file) {
+ Enumeration<JarEntry> entries = file.entries();
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ if (entry.isDirectory() || entry.getName().equals(JarFile.MANIFEST_NAME)) {
+ continue;
+ }
+ resources.get(classloader).add(entry.getName());
+ }
+ }
+
+ @Override
+ protected void scanDirectory(ClassLoader classloader, File directory) throws IOException {
+ scanDirectory(directory, classloader, "");
+ }
+
+ private void scanDirectory(File directory, ClassLoader classloader, String packagePrefix)
+ throws IOException {
+ File[] files = directory.listFiles();
+ if (files == null) {
+ logger.warn("Cannot read directory " + directory);
+ // IO error, just skip the directory
+ return;
+ }
+ for (File f : files) {
+ String name = f.getName();
+ if (f.isDirectory()) {
+ scanDirectory(f, classloader, packagePrefix + name + "/");
+ } else {
+ String resourceName = packagePrefix + name;
+ if (!resourceName.equals(JarFile.MANIFEST_NAME)) {
+ resources.get(classloader).add(resourceName);
+ }
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ static String getClassName(String filename) {
+ int classNameEnd = filename.length() - CLASS_FILE_NAME_EXTENSION.length();
+ return filename.substring(0, classNameEnd).replace('/', '.');
+ }
+
+ @VisibleForTesting
+ static File toFile(URL url) {
+ checkArgument(url.getProtocol().equals("file"));
+ try {
+ return new File(url.toURI()); // Accepts escaped characters like %20.
+ } catch (URISyntaxException e) { // URL.toURI() doesn't escape chars.
+ return new File(url.getPath()); // Accepts non-escaped chars like space.
+ }
+ }
+}
[2/2] beam git commit: This closes #2799
Posted by dh...@apache.org.
This closes #2799
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c2a55578
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c2a55578
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c2a55578
Branch: refs/heads/master
Commit: c2a5557839c6146e5a3f386172d4d068c67ac656
Parents: 3ae9441 0788d3d
Author: Dan Halperin <dh...@google.com>
Authored: Mon May 1 14:23:49 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon May 1 14:23:49 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/beam/findbugs-filter.xml | 6 +
.../org/apache/beam/sdk/util/ApiSurface.java | 4 +-
.../org/apache/beam/sdk/util/ClassPath.java | 544 +++++++++++++++++++
3 files changed, 551 insertions(+), 3 deletions(-)
----------------------------------------------------------------------