You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/01/15 11:53:34 UTC
[04/12] flink git commit: [hotfix] Remove unnecessary dependencies in
flink-runtime.
[hotfix] Remove unnecessary dependencies in flink-runtime.
- Removes Apache Commons HTTP Client
- Removes Apache Commons IO
- Removes Jettison JSONparser and consolidates tests to use Jackson instead (which is included anyways)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/388c280f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/388c280f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/388c280f
Branch: refs/heads/master
Commit: 388c280fcd8157f5ebc06a9062a0d06a3d2e92cb
Parents: a65cd8d
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Jan 6 10:22:00 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Jan 15 11:44:20 2016 +0100
----------------------------------------------------------------------
.../main/resources/archetype-resources/pom.xml | 11 ---
.../main/resources/archetype-resources/pom.xml | 13 ---
flink-runtime/pom.xml | 17 ----
.../runtime/webmonitor/WebMonitorUtils.java | 37 +++++----
.../flink/test/web/WebFrontendITCase.java | 85 ++++++++++++--------
.../flink/yarn/YARNSessionFIFOITCase.java | 29 ++++---
tools/maven/checkstyle.xml | 6 ++
7 files changed, 99 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 1610a3a..4baf668 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -139,13 +139,6 @@ under the License.
<exclude>com.typesafe.akka:akka-slf4j_*</exclude>
<exclude>io.netty:netty-all</exclude>
<exclude>io.netty:netty</exclude>
- <exclude>org.eclipse.jetty:jetty-server</exclude>
- <exclude>org.eclipse.jetty:jetty-continuation</exclude>
- <exclude>org.eclipse.jetty:jetty-http</exclude>
- <exclude>org.eclipse.jetty:jetty-io</exclude>
- <exclude>org.eclipse.jetty:jetty-util</exclude>
- <exclude>org.eclipse.jetty:jetty-security</exclude>
- <exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>commons-fileupload:commons-fileupload</exclude>
<exclude>org.apache.avro:avro</exclude>
<exclude>commons-collections:commons-collections</exclude>
@@ -175,18 +168,14 @@ under the License.
<exclude>org.apache.commons:commons-math</exclude>
<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
<exclude>commons-logging:commons-logging</exclude>
- <exclude>org.apache.httpcomponents:httpclient</exclude>
- <exclude>org.apache.httpcomponents:httpcore</exclude>
<exclude>commons-codec:commons-codec</exclude>
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
- <exclude>org.codehaus.jettison:jettison</exclude>
<exclude>stax:stax-api</exclude>
<exclude>com.typesafe:config</exclude>
<exclude>org.uncommons.maths:uncommons-maths</exclude>
<exclude>com.github.scopt:scopt_*</exclude>
- <exclude>org.mortbay.jetty:servlet-api</exclude>
<exclude>commons-io:commons-io</exclude>
<exclude>commons-cli:commons-cli</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index e8c9057..560a29f 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -144,18 +144,9 @@ under the License.
<exclude>com.typesafe.akka:akka-slf4j_*</exclude>
<exclude>io.netty:netty-all</exclude>
<exclude>io.netty:netty</exclude>
- <exclude>org.eclipse.jetty:jetty-server</exclude>
- <exclude>org.eclipse.jetty:jetty-continuation</exclude>
- <exclude>org.eclipse.jetty:jetty-http</exclude>
- <exclude>org.eclipse.jetty:jetty-io</exclude>
- <exclude>org.eclipse.jetty:jetty-util</exclude>
- <exclude>org.eclipse.jetty:jetty-security</exclude>
- <exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>commons-fileupload:commons-fileupload</exclude>
<exclude>org.apache.avro:avro</exclude>
<exclude>commons-collections:commons-collections</exclude>
- <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
- <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
<exclude>com.thoughtworks.paranamer:paranamer</exclude>
<exclude>org.xerial.snappy:snappy-java</exclude>
<exclude>org.apache.commons:commons-compress</exclude>
@@ -180,18 +171,14 @@ under the License.
<exclude>org.apache.commons:commons-math</exclude>
<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
<exclude>commons-logging:commons-logging</exclude>
- <exclude>org.apache.httpcomponents:httpclient</exclude>
- <exclude>org.apache.httpcomponents:httpcore</exclude>
<exclude>commons-codec:commons-codec</exclude>
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
- <exclude>org.codehaus.jettison:jettison</exclude>
<exclude>stax:stax-api</exclude>
<exclude>com.typesafe:config</exclude>
<exclude>org.uncommons.maths:uncommons-maths</exclude>
<exclude>com.github.scopt:scopt_*</exclude>
- <exclude>org.mortbay.jetty:servlet-api</exclude>
<exclude>commons-io:commons-io</exclude>
<exclude>commons-cli:commons-cli</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index c3093da..7feace8 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -71,17 +71,6 @@ under the License.
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <!-- Version is set in root POM -->
- </dependency>
-
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<!-- Version is set in root POM -->
@@ -95,12 +84,6 @@ under the License.
</dependency>
<dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- <version>1.1</version>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
index 41cb83b..6d89de0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
@@ -20,6 +20,10 @@ package org.apache.flink.runtime.webmonitor;
import akka.actor.ActorSystem;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.execution.ExecutionState;
@@ -30,10 +34,6 @@ import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.messages.webmonitor.JobDetails;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +41,7 @@ import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
/**
@@ -142,17 +143,25 @@ public final class WebMonitorUtils {
}
}
- public static Map<String, String> fromKeyValueJsonArray(JSONArray parsed) throws JSONException {
- Map<String, String> hashMap = new HashMap<>();
-
- for (int i = 0; i < parsed.length(); i++) {
- JSONObject jsonObject = parsed.getJSONObject(i);
- String key = jsonObject.getString("key");
- String value = jsonObject.getString("value");
- hashMap.put(key, value);
+ public static Map<String, String> fromKeyValueJsonArray(String jsonString) {
+ try {
+ Map<String, String> map = new HashMap<>();
+ ObjectMapper m = new ObjectMapper();
+ ArrayNode array = (ArrayNode) m.readTree(jsonString);
+
+ Iterator<JsonNode> elements = array.elements();
+ while (elements.hasNext()) {
+ JsonNode node = elements.next();
+ String key = node.get("key").asText();
+ String value = node.get("value").asText();
+ map.put(key, value);
+ }
+
+ return map;
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e.getMessage(), e);
}
-
- return hashMap;
}
public static JobDetails createDetailsForJob(ExecutionGraph job) {
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
index 7720afd..30f840c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java
@@ -18,6 +18,10 @@
package org.apache.flink.test.web;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+
import org.apache.commons.io.FileUtils;
import org.apache.flink.configuration.ConfigConstants;
@@ -25,9 +29,6 @@ import org.apache.flink.runtime.webmonitor.WebMonitor;
import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.apache.flink.test.util.TestBaseUtils;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -38,6 +39,11 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
@RunWith(Parameterized.class)
public class WebFrontendITCase extends MultipleProgramsTestBase {
@@ -69,10 +75,11 @@ public class WebFrontendITCase extends MultipleProgramsTestBase {
try {
String fromHTTP = TestBaseUtils.getFromHTTP("http://localhost:" + port + "/index.html");
String text = "Apache Flink Dashboard";
- Assert.assertTrue("Startpage should contain " + text, fromHTTP.contains(text));
- } catch (Exception e) {
+ assertTrue("Startpage should contain " + text, fromHTTP.contains(text));
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -80,13 +87,17 @@ public class WebFrontendITCase extends MultipleProgramsTestBase {
public void getNumberOfTaskManagers() {
try {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + port + "/taskmanagers/");
- JSONObject response = new JSONObject(json);
- JSONArray taskManagers = response.getJSONArray("taskmanagers");
- Assert.assertNotNull(taskManagers);
- Assert.assertEquals(cluster.numTaskManagers(), taskManagers.length());
- }catch(Throwable e) {
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode response = mapper.readTree(json);
+ ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");
+
+ assertNotNull(taskManagers);
+ assertEquals(cluster.numTaskManagers(), taskManagers.size());
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -94,16 +105,21 @@ public class WebFrontendITCase extends MultipleProgramsTestBase {
public void getTaskmanagers() {
try {
String json = getFromHTTP("http://localhost:" + port + "/taskmanagers/");
- JSONObject parsed = new JSONObject(json);
- JSONArray taskManagers = parsed.getJSONArray("taskmanagers");
- Assert.assertNotNull(taskManagers);
- Assert.assertEquals(cluster.numTaskManagers(), taskManagers.length());
- JSONObject taskManager = taskManagers.getJSONObject(0);
- Assert.assertNotNull(taskManager);
- Assert.assertEquals(4, taskManager.getInt("freeSlots"));
- }catch(Throwable e) {
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode parsed = mapper.readTree(json);
+ ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
+
+ assertNotNull(taskManagers);
+ assertEquals(cluster.numTaskManagers(), taskManagers.size());
+
+ JsonNode taskManager = taskManagers.get(0);
+ assertNotNull(taskManager);
+ assertEquals(4, taskManager.get("freeSlots").asInt());
+ }
+ catch(Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -114,14 +130,15 @@ public class WebFrontendITCase extends MultipleProgramsTestBase {
FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
String logs = getFromHTTP("http://localhost:" + port + "/jobmanager/log");
- Assert.assertTrue(logs.contains("job manager log"));
+ assertTrue(logs.contains("job manager log"));
FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
logs = getFromHTTP("http://localhost:" + port + "/jobmanager/stdout");
- Assert.assertTrue(logs.contains("job manager out"));
- } catch(Throwable e) {
+ assertTrue(logs.contains("job manager out"));
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -129,16 +146,16 @@ public class WebFrontendITCase extends MultipleProgramsTestBase {
public void getConfiguration() {
try {
String config = getFromHTTP("http://localhost:" + port + "/jobmanager/config");
- JSONArray array = new JSONArray(config);
-
- Map<String, String> conf = WebMonitorUtils.fromKeyValueJsonArray(array);
- Assert.assertTrue(conf.get(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY).startsWith(logDir.toString()));
- Assert.assertEquals(
- cluster.configuration().getString("taskmanager.numberOfTaskSlots", null),
- conf.get(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS));
- } catch (Throwable e) {
+
+ Map<String, String> conf = WebMonitorUtils.fromKeyValueJsonArray(config);
+ assertTrue(conf.get(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY).startsWith(logDir.toString()));
+ assertEquals(
+ cluster.configuration().getString("taskmanager.numberOfTaskSlots", null),
+ conf.get(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS));
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 69264cd..8c9a9c7 100644
--- a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -15,10 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.yarn;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.common.base.Joiner;
+
import org.apache.commons.io.FileUtils;
+
import org.apache.flink.client.FlinkYarnSessionCli;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.GlobalConfiguration;
@@ -29,6 +35,7 @@ import org.apache.flink.runtime.yarn.AbstractFlinkYarnCluster;
import org.apache.flink.runtime.yarn.FlinkYarnClusterStatus;
import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.TestBaseUtils;
+
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
@@ -43,14 +50,15 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+
import org.apache.log4j.Level;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -199,16 +207,17 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
LOG.info("Got application URL from YARN {}", url);
String response = TestBaseUtils.getFromHTTP(url + "taskmanagers/");
- JSONObject parsedTMs = new JSONObject(response);
- JSONArray taskManagers = parsedTMs.getJSONArray("taskmanagers");
+
+
+ JsonNode parsedTMs = new ObjectMapper().readTree(response);
+ ArrayNode taskManagers = (ArrayNode) parsedTMs.get("taskmanagers");
Assert.assertNotNull(taskManagers);
- Assert.assertEquals(1, taskManagers.length());
- Assert.assertEquals(1, taskManagers.getJSONObject(0).getInt("slotsNumber"));
+ Assert.assertEquals(1, taskManagers.size());
+ Assert.assertEquals(1, taskManagers.get(0).get("slotsNumber").asInt());
// get the configuration from webinterface & check if the dynamic properties from YARN show up there.
String jsonConfig = TestBaseUtils.getFromHTTP(url + "jobmanager/config");
- JSONArray parsed = new JSONArray(jsonConfig);
- Map<String, String> parsedConfig = WebMonitorUtils.fromKeyValueJsonArray(parsed);
+ Map<String, String> parsedConfig = WebMonitorUtils.fromKeyValueJsonArray(jsonConfig);
Assert.assertEquals("veryFancy", parsedConfig.get("fancy-configuration-value"));
Assert.assertEquals("3", parsedConfig.get("yarn.maximum-failed-containers"));
@@ -226,9 +235,9 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
}
LOG.info("Extracted hostname:port: {} {}", hostname, port);
- Assert.assertEquals("unable to find hostname in " + parsed, hostname,
+ Assert.assertEquals("unable to find hostname in " + jsonConfig, hostname,
parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY));
- Assert.assertEquals("unable to find port in " + parsed, port,
+ Assert.assertEquals("unable to find port in " + jsonConfig, port,
parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY));
// test logfile access
http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/tools/maven/checkstyle.xml
----------------------------------------------------------------------
diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml
index a558116..d5ee277 100644
--- a/tools/maven/checkstyle.xml
+++ b/tools/maven/checkstyle.xml
@@ -76,6 +76,12 @@ under the License.
<property name="illegalPattern" value="true"/>
<property name="message" value="Use commons-lang3 instead of commons-lang."/>
</module>
+ <module name="Regexp">
+ <property name="format" value="org\.codehaus\.jettison"/>
+ <property name="illegalPattern" value="true"/>
+ <property name="message" value="Use com.fasterxml.jackson instead of jettison."/>
+ </module>
+
<module name="NeedBraces">
<property name="tokens" value="LITERAL_IF, LITERAL_ELSE"/>
</module>