You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2016/10/11 01:06:08 UTC
[1/2] apex-core git commit: APEXCORE-536 #resolve Upgrade Hadoop
dependency version from 2.2.0 to 2.6.0
Repository: apex-core
Updated Branches:
refs/heads/master edad2a848 -> 5adcd0367
APEXCORE-536 #resolve Upgrade Hadoop dependency version from 2.2.0 to 2.6.0
Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/730b283f
Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/730b283f
Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/730b283f
Branch: refs/heads/master
Commit: 730b283f21e89b0840144a371f8586236e2fc4de
Parents: 0bdf771
Author: David Yan <da...@datatorrent.com>
Authored: Mon Oct 3 18:03:59 2016 -0700
Committer: David Yan <da...@datatorrent.com>
Committed: Mon Oct 10 13:17:07 2016 -0700
----------------------------------------------------------------------
engine/pom.xml | 29 +++++++++++++++-----
.../java/com/datatorrent/stram/StramClient.java | 16 ++++-------
.../stram/client/StramClientUtils.java | 12 ++++----
pom.xml | 2 +-
4 files changed, 35 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-core/blob/730b283f/engine/pom.xml
----------------------------------------------------------------------
diff --git a/engine/pom.xml b/engine/pom.xml
index 59c096e..4e005fb 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -181,8 +181,8 @@
<version>0.5</version>
<exclusions>
<exclusion>
- <artifactId>commons-beanutils-core</artifactId>
<groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -217,12 +217,12 @@
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
- <artifactId>jackson-jaxrs</artifactId>
<groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
- <artifactId>jackson-xc</artifactId>
<groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -234,8 +234,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
@@ -268,12 +268,12 @@
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
- <artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
</exclusion>
<exclusion>
- <artifactId>commons-beanutils-core</artifactId>
<groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -301,8 +301,8 @@
<type>jar</type>
<exclusions>
<exclusion>
- <artifactId>commons-collections</artifactId>
<groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -388,6 +388,21 @@
<version>1.6.5</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <version>1.9</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <!-- This is to prevent javax.servlet artifact to be pulled in
+ as a transitive dependency during the unit tests since this
+ conflicts with org.eclipse.jetty:jetty-servlet. -->
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <version>2.5</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/apex-core/blob/730b283f/engine/src/main/java/com/datatorrent/stram/StramClient.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java
index 5094dee..fe820b1 100644
--- a/engine/src/main/java/com/datatorrent/stram/StramClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java
@@ -123,7 +123,7 @@ public class StramClient
// platform dependencies that are not part of Hadoop and need to be deployed,
// entry below will cause containing jar file from client to be copied to cluster
- private static final Class<?>[] DATATORRENT_CLASSES = new Class<?>[]{
+ private static final Class<?>[] APEX_CLASSES = new Class<?>[]{
com.datatorrent.netlet.util.Slice.class,
com.datatorrent.netlet.EventLoop.class,
com.datatorrent.bufferserver.server.Server.class,
@@ -144,18 +144,14 @@ public class StramClient
org.apache.http.message.BasicHeaderValueParser.class,
com.esotericsoftware.minlog.Log.class,
org.apache.xbean.asm5.tree.ClassNode.class,
- // The jersey client inclusion is only for Hadoop-2.2 and should be removed when we upgrade our Hadoop
- // dependency version since Hadoop-2.3 onwards has jersey client bundled
- com.sun.jersey.api.client.ClientHandler.class,
- com.sun.jersey.client.apache4.ApacheHttpClient4Handler.class,
org.jctools.queues.SpscArrayQueue.class
};
- private static final Class<?>[] DATATORRENT_SECURITY_SPECIFIC_CLASSES = new Class<?>[]{
+ private static final Class<?>[] APEX_SECURITY_SPECIFIC_CLASSES = new Class<?>[]{
};
- private static final Class<?>[] DATATORRENT_SECURITY_CLASSES =
- (Class<?>[])ArrayUtils.addAll(DATATORRENT_CLASSES, DATATORRENT_SECURITY_SPECIFIC_CLASSES);
+ private static final Class<?>[] APEX_SECURITY_CLASSES =
+ (Class<?>[])ArrayUtils.addAll(APEX_CLASSES, APEX_SECURITY_SPECIFIC_CLASSES);
public StramClient(Configuration conf, LogicalPlan dag) throws Exception
{
@@ -325,9 +321,9 @@ public class StramClient
if (applicationType.equals(YARN_APPLICATION_TYPE_DEPRECATED)) {
//TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
- defaultClasses = DATATORRENT_SECURITY_CLASSES;
+ defaultClasses = APEX_SECURITY_CLASSES;
} else {
- defaultClasses = DATATORRENT_CLASSES;
+ defaultClasses = APEX_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
http://git-wip-us.apache.org/repos/asf/apex-core/blob/730b283f/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
index c7e3756..5ce0e4a 100644
--- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
+++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
@@ -106,7 +106,7 @@ public class StramClientUtils
public static final String SUBDIR_APPS = "apps";
public static final String SUBDIR_PROFILES = "profiles";
public static final String SUBDIR_CONF = "conf";
- public static final int RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE = 10 * 1000;
+ public static final long RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE = 10 * 1000;
public static final String DT_HDFS_TOKEN_MAX_LIFE_TIME = StreamingApplication.DT_PREFIX + "namenode.delegation.token.max-lifetime";
public static final String HDFS_TOKEN_MAX_LIFE_TIME = "dfs.namenode.delegation.token.max-lifetime";
public static final String DT_RM_TOKEN_MAX_LIFE_TIME = StreamingApplication.DT_PREFIX + "resourcemanager.delegation.token.max-lifetime";
@@ -432,15 +432,15 @@ public class StramClientUtils
// We are overriding this to be 10 seconds maximum.
//
- int rmConnectMaxWait = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
+ long rmConnectMaxWait = conf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
if (rmConnectMaxWait > RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE) {
LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.", YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmConnectMaxWait, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE);
- conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE);
- int rmConnectRetryInterval = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
- int defaultRetryInterval = Math.max(500, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE / 5);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE);
+ long rmConnectRetryInterval = conf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
+ long defaultRetryInterval = Math.max(500, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE / 5);
if (rmConnectRetryInterval > defaultRetryInterval) {
LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.", YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, rmConnectRetryInterval, defaultRetryInterval);
- conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, defaultRetryInterval);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, defaultRetryInterval);
}
}
LOG.info(" conf object in stramclient {}", conf);
http://git-wip-us.apache.org/repos/asf/apex-core/blob/730b283f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6107df7..e4c57a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.optimize>false</maven.compiler.optimize>
<maven.build.timestamp.format>yyyyMMdd</maven.build.timestamp.format>
- <hadoop.version>2.2.0</hadoop.version>
+ <hadoop.version>2.6.0</hadoop.version>
<github.global.server>github</github.global.server>
<jackson.version>1.9.13</jackson.version>
<jersey.version>1.9</jersey.version>
[2/2] apex-core git commit: Merge branch 'APEXCORE-536' of
https://github.com/davidyan74/apex-core
Posted by th...@apache.org.
Merge branch 'APEXCORE-536' of https://github.com/davidyan74/apex-core
Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/5adcd036
Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/5adcd036
Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/5adcd036
Branch: refs/heads/master
Commit: 5adcd0367b3a702323e629fe0a92fdbfffb2c9bd
Parents: edad2a8 730b283
Author: Thomas Weise <th...@apache.org>
Authored: Mon Oct 10 17:27:30 2016 -0700
Committer: Thomas Weise <th...@apache.org>
Committed: Mon Oct 10 17:27:30 2016 -0700
----------------------------------------------------------------------
engine/pom.xml | 29 +++++++++++++++-----
.../java/com/datatorrent/stram/StramClient.java | 16 ++++-------
.../stram/client/StramClientUtils.java | 12 ++++----
pom.xml | 2 +-
4 files changed, 35 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-core/blob/5adcd036/engine/src/main/java/com/datatorrent/stram/StramClient.java
----------------------------------------------------------------------
diff --cc engine/src/main/java/com/datatorrent/stram/StramClient.java
index 2009b49,fe820b1..89bca14
--- a/engine/src/main/java/com/datatorrent/stram/StramClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java
@@@ -323,12 -318,12 +319,12 @@@ public class StramClien
{
Class<?>[] defaultClasses;
- if (applicationType.equals(YARN_APPLICATION_TYPE_DEPRECATED)) {
+ if (applicationType.equals(YARN_APPLICATION_TYPE)) {
//TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
- defaultClasses = DATATORRENT_SECURITY_CLASSES;
+ defaultClasses = APEX_SECURITY_CLASSES;
} else {
- defaultClasses = DATATORRENT_CLASSES;
+ defaultClasses = APEX_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
http://git-wip-us.apache.org/repos/asf/apex-core/blob/5adcd036/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java
----------------------------------------------------------------------