You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by th...@apache.org on 2017/04/08 04:46:47 UTC
[1/3] beam git commit: Fix for potentially unclosed streams in
ApexYarnLauncher
Repository: beam
Updated Branches:
refs/heads/master 0c3dc0051 -> 010d2882a
Fix for potentially unclosed streams in ApexYarnLauncher
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ca4fae4d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ca4fae4d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ca4fae4d
Branch: refs/heads/master
Commit: ca4fae4d26da8c222209352dca35b1f54a6263de
Parents: f87597e
Author: rjoshi2 <re...@gmail.com>
Authored: Fri Apr 7 18:12:09 2017 -0700
Committer: rjoshi2 <re...@gmail.com>
Committed: Fri Apr 7 18:12:09 2017 -0700
----------------------------------------------------------------------
.../beam/runners/apex/ApexYarnLauncher.java | 103 ++++++++++---------
1 file changed, 52 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ca4fae4d/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
index 6bc42f0..e162d85 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
@@ -173,56 +173,57 @@ public class ApexYarnLauncher {
* @throws IOException when dependency information cannot be read
*/
public static List<File> getYarnDeployDependencies() throws IOException {
- InputStream dependencyTree = ApexRunner.class.getResourceAsStream("dependency-tree");
- BufferedReader br = new BufferedReader(new InputStreamReader(dependencyTree));
- String line = null;
- List<String> excludes = new ArrayList<>();
- int excludeLevel = Integer.MAX_VALUE;
- while ((line = br.readLine()) != null) {
- for (int i = 0; i < line.length(); i++) {
- char c = line.charAt(i);
- if (Character.isLetter(c)) {
- if (i > excludeLevel) {
- excludes.add(line.substring(i));
- } else {
- if (line.substring(i).startsWith("org.apache.hadoop")) {
- excludeLevel = i;
- excludes.add(line.substring(i));
- } else {
- excludeLevel = Integer.MAX_VALUE;
+ try (InputStream dependencyTree = ApexRunner.class.getResourceAsStream("dependency-tree")) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(dependencyTree))) {
+ String line;
+ List<String> excludes = new ArrayList<>();
+ int excludeLevel = Integer.MAX_VALUE;
+ while ((line = br.readLine()) != null) {
+ for (int i = 0; i < line.length(); i++) {
+ char c = line.charAt(i);
+ if (Character.isLetter(c)) {
+ if (i > excludeLevel) {
+ excludes.add(line.substring(i));
+ } else {
+ if (line.substring(i).startsWith("org.apache.hadoop")) {
+ excludeLevel = i;
+ excludes.add(line.substring(i));
+ } else {
+ excludeLevel = Integer.MAX_VALUE;
+ }
+ }
+ break;
}
}
- break;
}
- }
- }
- br.close();
-
- Set<String> excludeJarFileNames = Sets.newHashSet();
- for (String exclude : excludes) {
- String[] mvnc = exclude.split(":");
- String fileName = mvnc[1] + "-";
- if (mvnc.length == 6) {
- fileName += mvnc[4] + "-" + mvnc[3]; // with classifier
- } else {
- fileName += mvnc[3];
- }
- fileName += ".jar";
- excludeJarFileNames.add(fileName);
- }
- ClassLoader classLoader = ApexYarnLauncher.class.getClassLoader();
- URL[] urls = ((URLClassLoader) classLoader).getURLs();
- List<File> dependencyJars = new ArrayList<>();
- for (int i = 0; i < urls.length; i++) {
- File f = new File(urls[i].getFile());
- // dependencies can also be directories in the build reactor,
- // the Apex client will automatically create jar files for those.
- if (f.exists() && !excludeJarFileNames.contains(f.getName())) {
- dependencyJars.add(f);
+ Set<String> excludeJarFileNames = Sets.newHashSet();
+ for (String exclude : excludes) {
+ String[] mvnc = exclude.split(":");
+ String fileName = mvnc[1] + "-";
+ if (mvnc.length == 6) {
+ fileName += mvnc[4] + "-" + mvnc[3]; // with classifier
+ } else {
+ fileName += mvnc[3];
+ }
+ fileName += ".jar";
+ excludeJarFileNames.add(fileName);
+ }
+
+ ClassLoader classLoader = ApexYarnLauncher.class.getClassLoader();
+ URL[] urls = ((URLClassLoader) classLoader).getURLs();
+ List<File> dependencyJars = new ArrayList<>();
+ for (int i = 0; i < urls.length; i++) {
+ File f = new File(urls[i].getFile());
+ // dependencies can also be directories in the build reactor,
+ // the Apex client will automatically create jar files for those.
+ if (f.exists() && !excludeJarFileNames.contains(f.getName())) {
+ dependencyJars.add(f);
+ }
+ }
+ return dependencyJars;
}
}
- return dependencyJars;
}
/**
@@ -238,17 +239,17 @@ public class ApexYarnLauncher {
throw new RuntimeException("Failed to remove " + jarFile);
}
URI uri = URI.create("jar:" + jarFile.toURI());
- try (final FileSystem zipfs = FileSystems.newFileSystem(uri, env);) {
+ try (final FileSystem zipfs = FileSystems.newFileSystem(uri, env)) {
File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
Files.createDirectory(zipfs.getPath("META-INF"));
- final OutputStream out = Files.newOutputStream(zipfs.getPath(JarFile.MANIFEST_NAME));
- if (!manifestFile.exists()) {
- new Manifest().write(out);
- } else {
- FileUtils.copyFile(manifestFile, out);
+ try (final OutputStream out = Files.newOutputStream(zipfs.getPath(JarFile.MANIFEST_NAME))) {
+ if (!manifestFile.exists()) {
+ new Manifest().write(out);
+ } else {
+ FileUtils.copyFile(manifestFile, out);
+ }
}
- out.close();
final java.nio.file.Path root = dir.toPath();
Files.walkFileTree(root, new java.nio.file.SimpleFileVisitor<Path>() {
[3/3] beam git commit: This closes #2469
Posted by th...@apache.org.
This closes #2469
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/010d2882
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/010d2882
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/010d2882
Branch: refs/heads/master
Commit: 010d2882aba6688e603964c85967ef90389f5c3d
Parents: 0c3dc00 564c17a
Author: Thomas Weise <th...@apache.org>
Authored: Fri Apr 7 21:46:23 2017 -0700
Committer: Thomas Weise <th...@apache.org>
Committed: Fri Apr 7 21:46:23 2017 -0700
----------------------------------------------------------------------
.../beam/runners/apex/ApexYarnLauncher.java | 109 ++++++++++---------
1 file changed, 55 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
[2/3] beam git commit: Fix for potentially unclosed streams in
ApexYarnLauncher
Posted by th...@apache.org.
Fix for potentially unclosed streams in ApexYarnLauncher
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/564c17ac
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/564c17ac
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/564c17ac
Branch: refs/heads/master
Commit: 564c17acfc966b6f66c36b3eac0f9f76794b7d4d
Parents: ca4fae4
Author: rjoshi2 <re...@gmail.com>
Authored: Fri Apr 7 19:15:12 2017 -0700
Committer: rjoshi2 <re...@gmail.com>
Committed: Fri Apr 7 19:15:12 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/runners/apex/ApexYarnLauncher.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/564c17ac/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
----------------------------------------------------------------------
diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
index e162d85..198b9bf 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexYarnLauncher.java
@@ -275,9 +275,9 @@ public class ApexYarnLauncher {
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String name = relativePath + file.getFileName();
if (!JarFile.MANIFEST_NAME.equals(name)) {
- final OutputStream out = Files.newOutputStream(zipfs.getPath(name));
- FileUtils.copyFile(file.toFile(), out);
- out.close();
+ try (final OutputStream out = Files.newOutputStream(zipfs.getPath(name))) {
+ FileUtils.copyFile(file.toFile(), out);
+ }
}
return super.visitFile(file, attrs);
}