You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/27 05:57:35 UTC
bigtop git commit: BIGTOP-2706 Exclude hadoop dependencies from apex
packaging for 3.5.0
Repository: bigtop
Updated Branches:
refs/heads/master 611a101aa -> c149934cd
BIGTOP-2706 Exclude hadoop dependencies from apex packaging for 3.5.0
Closes #188
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c149934c
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c149934c
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c149934c
Branch: refs/heads/master
Commit: c149934cd9779f01eb6632cd5e4432e1ee8f58ff
Parents: 611a101
Author: Chinmay Kolhatkar <ch...@apache.org>
Authored: Tue Mar 21 18:43:05 2017 +0530
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Sun Mar 26 22:56:52 2017 -0700
----------------------------------------------------------------------
.../src/common/apex/do-component-build | 55 ++++++++++++++++++--
.../src/common/apex/patch1-for-hadoop27x.diff | 24 +++++++++
bigtop-packages/src/rpm/apex/SPECS/apex.spec | 3 ++
3 files changed, 77 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/common/apex/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/apex/do-component-build b/bigtop-packages/src/common/apex/do-component-build
index b940275..49a8e82 100644
--- a/bigtop-packages/src/common/apex/do-component-build
+++ b/bigtop-packages/src/common/apex/do-component-build
@@ -26,10 +26,10 @@ command -v mvn > /dev/null 2>&1 || {
}
# Build apex
-mvn clean package -DskipTests "$@"
+mvn clean package -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@"
# Find all the dependencies of apex
-mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/all-deps -DincludeScope=runtime -DexcludeScope=test
+mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/all-deps -DincludeScope=runtime -DexcludeScope=test -Dhadoop.version=$HADOOP_VERSION
# Copy apex-engine jar from compiled path
cp engine/target/apex-engine.jar all-deps/
@@ -50,12 +50,56 @@ cat > pom-hadoop-deps.xml <<'EOF'
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>2.2.0</version>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.2.0</version>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-beanutils-core</artifactId>
+ <groupId>commons-beanutils</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-jaxrs</artifactId>
+ <groupId>org.codehaus.jackson</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-xc</artifactId>
+ <groupId>org.codehaus.jackson</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-server</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
</project>
@@ -64,6 +108,7 @@ EOF
mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/hadoop-deps \
-DincludeScope=runtime -DexcludeScope=test \
-DexcludeArtifactIds=jersey-client \
+ -Dhadoop.version=$HADOOP_VERSION \
-f pom-hadoop-deps.xml
# Find the diff between platform and hadoop. And that's what is required for running apex in standalone mode.
@@ -71,7 +116,7 @@ mkdir deps
for i in `diff -r all-deps/ hadoop-deps/ | grep all-deps | awk '{print $4}'`; do cp all-deps/$i deps/; done
rm -rf all-deps/ hadoop-deps/
-rm -rf deps/hadoop-common-2.2.0-tests.jar
+rm -rf deps/hadoop-common-$HADOOP_VERSION-tests.jar
#Download non-ASF licenses for libraries
wget -t 3 -q -O jline-LICENSE.txt https://raw.githubusercontent.com/jline/jline2/jline-2.11/LICENSE.txt
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff b/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff
new file mode 100644
index 0000000..2cd6be7
--- /dev/null
+++ b/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff
@@ -0,0 +1,24 @@
+diff -Naur orig/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java new/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java
+--- orig/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java 2016-11-22 02:49:11.000000000 +0530
++++ new/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java 2017-03-23 23:08:32.692947303 +0530
+@@ -583,7 +583,7 @@
+ if (UserGroupInformation.isSecurityEnabled()) {
+ long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
+ dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
+- long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
++ long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
+ dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
+ setTokenRefreshCredentials(dag, conf);
+ }
+diff -Naur orig/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java new/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java
+--- orig/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java 2016-11-22 02:48:16.000000000 +0530
++++ new/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java 2017-03-23 23:07:18.948865455 +0530
+@@ -156,7 +156,7 @@
+ */
+ public static Attribute<Boolean> FAST_PUBLISHER_SUBSCRIBER = new Attribute<>(false);
+ public static Attribute<Long> HDFS_TOKEN_LIFE_TIME = new Attribute<>(604800000L);
+- public static Attribute<Long> RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
++ public static Attribute<Long> RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+ public static Attribute<String> PRINCIPAL = new Attribute<String>(null, new StringCodec.String2String());
+ public static Attribute<String> KEY_TAB_FILE = new Attribute<>((String)null, new StringCodec.String2String());
+ public static Attribute<Double> TOKEN_REFRESH_ANTICIPATORY_FACTOR = new Attribute<>(0.7);
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/rpm/apex/SPECS/apex.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/apex/SPECS/apex.spec b/bigtop-packages/src/rpm/apex/SPECS/apex.spec
index 210f2e9..3b290e8 100644
--- a/bigtop-packages/src/rpm/apex/SPECS/apex.spec
+++ b/bigtop-packages/src/rpm/apex/SPECS/apex.spec
@@ -44,6 +44,7 @@ Source1: do-component-build
Source2: install_apex.sh
Source3: apex.1
Source4: bigtop.bom
+#BIGTOP_PATCH_FILES
Requires: hadoop-client, bigtop-utils >= 0.7
%description
@@ -59,6 +60,8 @@ Apache Apex includes following key features:
%prep
%setup -n apache-%{name}-core-%{apex_base_version}
+#BIGTOP_PATCH_COMMANDS
+
%build
env APEX_VERSION=%{apex_base_version} bash %{SOURCE1}