You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2018/02/26 02:38:10 UTC
[1/2] storm git commit: STORM-2965 Interpret wildcard in classpath
correctly when reading config from classpath
Repository: storm
Updated Branches:
refs/heads/master a38f0c5c5 -> 51613c78a
STORM-2965 Interpret wildcard in classpath correctly when reading config from classpath
* also fix version pattern in SimpleVersion which parses version like 0.10.3 correctly
* handle ZIP file as well when it is a part of classpath
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/afc0ae52
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/afc0ae52
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/afc0ae52
Branch: refs/heads/master
Commit: afc0ae52312aee670c396885514f76a606cc3dfb
Parents: dc4011c
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Feb 20 22:07:08 2018 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Feb 26 11:35:28 2018 +0900
----------------------------------------------------------------------
.../org/apache/storm/utils/SimpleVersion.java | 2 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 128 +++++++++++++++----
.../apache/storm/utils/SimpleVersionTest.java | 69 ++++++++++
3 files changed, 173 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/afc0ae52/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java b/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java
index 471b049..98e61df 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java
@@ -27,7 +27,7 @@ public class SimpleVersion implements Comparable <SimpleVersion> {
private final int _major;
private final int _minor;
- private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)[.-_]+(\\d+).*");
+ private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)[\\.\\-\\_]+(\\d+).*");
public SimpleVersion(String version) {
Matcher m = VERSION_PATTERN.matcher(version);
http://git-wip-us.apache.org/repos/asf/storm/blob/afc0ae52/storm-client/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/Utils.java b/storm-client/src/jvm/org/apache/storm/utils/Utils.java
index 26d267e..af03787 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/Utils.java
@@ -63,6 +63,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
@@ -82,7 +84,6 @@ import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.generated.TopologySummary;
-import org.apache.storm.generated.WorkerToken;
import org.apache.storm.security.auth.ReqContext;
import org.apache.storm.serialization.DefaultSerializationDelegate;
import org.apache.storm.serialization.SerializationDelegate;
@@ -1504,37 +1505,58 @@ public class Utils {
Yaml yaml = new Yaml(new SafeConstructor());
Map<String, Object> defaultsConf = null;
Map<String, Object> stormConf = null;
+
+ // Based on how Java handles the classpath
+ // https://docs.oracle.com/javase/8/docs/technotes/tools/unix/classpath.html
for (String part: cp) {
File f = new File(part);
- if (f.isDirectory()) {
+
+ if (f.getName().equals("*")) {
+ // wildcard is given in file
+ // in java classpath, '*' is expanded to all jar/JAR files in the directory
+ File dir = f.getParentFile();
+ if (dir == null) {
+ // it happens when part is just '*' rather than denoting some directory
+ dir = new File(".");
+ }
+
+ File[] jarFiles = dir.listFiles((dir1, name) -> name.endsWith(".jar") || name.endsWith(".JAR"));
+
+ // Quoting Javadoc in File.listFiles(FilenameFilter filter):
+ // Returns {@code null} if this abstract pathname does not denote a directory, or if an I/O error occurs.
+ // Both things are not expected and should not happen.
+ if (jarFiles == null) {
+ throw new IOException("Fail to list jar files in directory: " + dir);
+ }
+
+ for (File jarFile : jarFiles) {
+ JarConfigReader jarConfigReader = new JarConfigReader(yaml, defaultsConf, stormConf, jarFile).readJar();
+ defaultsConf = jarConfigReader.getDefaultsConf();
+ stormConf = jarConfigReader.getStormConf();
+ }
+ } else if (f.isDirectory()) {
+ // no wildcard, directory
if (defaultsConf == null) {
defaultsConf = readConfIgnoreNotFound(yaml, new File(f, "defaults.yaml"));
}
-
+
if (stormConf == null) {
stormConf = readConfIgnoreNotFound(yaml, new File(f, "storm.yaml"));
}
- } else {
- //Lets assume it is a jar file
- try (JarFile jarFile = new JarFile(f)) {
- Enumeration<JarEntry> jarEnums = jarFile.entries();
- while (jarEnums.hasMoreElements()) {
- JarEntry entry = jarEnums.nextElement();
- if (!entry.isDirectory()) {
- if (defaultsConf == null && entry.getName().equals("defaults.yaml")) {
- try (InputStream in = jarFile.getInputStream(entry)) {
- defaultsConf = (Map<String, Object>) yaml.load(new InputStreamReader(in));
- }
- }
-
- if (stormConf == null && entry.getName().equals("storm.yaml")) {
- try (InputStream in = jarFile.getInputStream(entry)) {
- stormConf = (Map<String, Object>) yaml.load(new InputStreamReader(in));
- }
- }
- }
- }
+ } else if (f.isFile()) {
+ // no wildcard, file
+ String fileName = f.getName();
+ if (fileName.endsWith(".zip") || fileName.endsWith(".ZIP")) {
+ JarConfigReader jarConfigReader = new JarConfigReader(yaml, defaultsConf, stormConf, f).readZip();
+ defaultsConf = jarConfigReader.getDefaultsConf();
+ stormConf = jarConfigReader.getStormConf();
+ } else if (fileName.endsWith(".jar") || fileName.endsWith(".JAR")) {
+ JarConfigReader jarConfigReader = new JarConfigReader(yaml, defaultsConf, stormConf, f).readJar();
+ defaultsConf = jarConfigReader.getDefaultsConf();
+ stormConf = jarConfigReader.getStormConf();
}
+ // Class path entries that are neither directories nor archives (.zip or JAR files)
+ // nor the asterisk (*) wildcard character are ignored.
}
}
if (stormConf != null) {
@@ -1559,8 +1581,8 @@ public class Utils {
Set<Integer> ids = srcMap.keySet();
Integer largestId = ids.stream().max(Integer::compareTo).get();
int end = largestId - start;
- ArrayList<V> result = new ArrayList<>(Collections.nCopies(end+1 , null)); // creates array[largestId+1] filled with nulls
- for( Map.Entry<Integer, V> entry : srcMap.entrySet() ) {
+ ArrayList<V> result = new ArrayList<>(Collections.nCopies(end + 1, null)); // creates array[largestId+1] filled with nulls
+ for (Map.Entry<Integer, V> entry : srcMap.entrySet()) {
int id = entry.getKey();
if (id < start) {
LOG.debug("Entry {} will be skipped it is too small {} ...", id, start);
@@ -1570,4 +1592,60 @@ public class Utils {
}
return result;
}
+
+ private static class JarConfigReader {
+ private Yaml yaml;
+ private Map<String, Object> defaultsConf;
+ private Map<String, Object> stormConf;
+ private File f;
+
+ public JarConfigReader(Yaml yaml, Map<String, Object> defaultsConf, Map<String, Object> stormConf, File f) {
+ this.yaml = yaml;
+ this.defaultsConf = defaultsConf;
+ this.stormConf = stormConf;
+ this.f = f;
+ }
+
+ public Map<String, Object> getDefaultsConf() {
+ return defaultsConf;
+ }
+
+ public Map<String, Object> getStormConf() {
+ return stormConf;
+ }
+
+ public JarConfigReader readZip() throws IOException {
+ try (ZipFile zipFile = new ZipFile(f)) {
+ readArchive(zipFile);
+ }
+ return this;
+ }
+
+ public JarConfigReader readJar() throws IOException {
+ try (JarFile jarFile = new JarFile(f)) {
+ readArchive(jarFile);
+ }
+ return this;
+ }
+
+ private void readArchive(ZipFile zipFile) throws IOException {
+ Enumeration<? extends ZipEntry> zipEnums = zipFile.entries();
+ while (zipEnums.hasMoreElements()) {
+ ZipEntry entry = zipEnums.nextElement();
+ if (!entry.isDirectory()) {
+ if (defaultsConf == null && entry.getName().equals("defaults.yaml")) {
+ try (InputStreamReader isr = new InputStreamReader(zipFile.getInputStream(entry))) {
+ defaultsConf = (Map<String, Object>) yaml.load(isr);
+ }
+ }
+
+ if (stormConf == null && entry.getName().equals("storm.yaml")) {
+ try (InputStreamReader isr = new InputStreamReader(zipFile.getInputStream(entry))) {
+ stormConf = (Map<String, Object>) yaml.load(isr);
+ }
+ }
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/afc0ae52/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java
----------------------------------------------------------------------
diff --git a/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java b/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java
new file mode 100644
index 0000000..056e22f
--- /dev/null
+++ b/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.storm.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SimpleVersionTest {
+
+ @Test
+ public void testParseStorm2x() {
+ SimpleVersion version = new SimpleVersion("2.1.2");
+ Assert.assertEquals(2, version.getMajor());
+ Assert.assertEquals(1, version.getMinor());
+ }
+
+ @Test
+ public void testParseStorm2xSnapshot() {
+ SimpleVersion version = new SimpleVersion("2.1.2-SNAPSHOT");
+ Assert.assertEquals(2, version.getMajor());
+ Assert.assertEquals(1, version.getMinor());
+ }
+
+ @Test
+ public void testParseStorm1x() {
+ SimpleVersion version = new SimpleVersion("1.0.4");
+ Assert.assertEquals(1, version.getMajor());
+ Assert.assertEquals(0, version.getMinor());
+ }
+
+ @Test
+ public void testParseStorm1xSnapshot() {
+ SimpleVersion version = new SimpleVersion("1.0.4-SNAPSHOT");
+ Assert.assertEquals(1, version.getMajor());
+ Assert.assertEquals(0, version.getMinor());
+ }
+
+ @Test
+ public void testParseStorm0x() {
+ SimpleVersion version = new SimpleVersion("0.10.3");
+ Assert.assertEquals(0, version.getMajor());
+ Assert.assertEquals(10, version.getMinor());
+ }
+
+ @Test
+ public void testParseStorm0xSnapshot() {
+ SimpleVersion version = new SimpleVersion("0.10.3-SNAPSHOT");
+ Assert.assertEquals(0, version.getMajor());
+ Assert.assertEquals(10, version.getMinor());
+ }
+
+}
\ No newline at end of file
[2/2] storm git commit: Merge branch 'STORM-2965' of
https://github.com/HeartSaVioR/storm into STORM-2965-merge
Posted by ka...@apache.org.
Merge branch 'STORM-2965' of https://github.com/HeartSaVioR/storm into STORM-2965-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/51613c78
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/51613c78
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/51613c78
Branch: refs/heads/master
Commit: 51613c78ae6060c398cefdf20fd7967084f9a55f
Parents: a38f0c5 afc0ae5
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Feb 26 11:37:57 2018 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Feb 26 11:37:57 2018 +0900
----------------------------------------------------------------------
.../org/apache/storm/utils/SimpleVersion.java | 2 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 128 +++++++++++++++----
.../apache/storm/utils/SimpleVersionTest.java | 69 ++++++++++
3 files changed, 173 insertions(+), 26 deletions(-)
----------------------------------------------------------------------