You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by gu...@apache.org on 2022/10/10 07:48:21 UTC
[bigtop] branch master updated: BIGTOP-3800: Add extra Ambari-metrics RPMs packages in Ambari build for Mpack (#1010)
This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 6af10a5e BIGTOP-3800: Add extra Ambari-metrics RPMs packages in Ambari build for Mpack (#1010)
6af10a5e is described below
commit 6af10a5e5c575c2c58f82160c27b027bd9f5eaa6
Author: Yuqi Gu <yu...@arm.com>
AuthorDate: Mon Oct 10 15:48:14 2022 +0800
BIGTOP-3800: Add extra Ambari-metrics RPMs packages in Ambari build for Mpack (#1010)
Ambari Metrics System: (AMS) collects, aggregates, and serves Hadoop
and system metrics in Ambari-managed clusters.
In Bigtop, we just offer Ambari-Server and Ambari-agent RPMs package,
no Ambari Metrics RPMs there.
Ambari Metrics, to be the sub-module of Ambari,
we could add extra Ambari-metrics RPMs packages in Ambari build for Mpack Ambari-metrics.
Ambari DO NOT support debian deployment in current version(2.7.5).
We just add RPMs support in Bigtop 3.2 release.
Change-Id: Ibca6f8d3a1727d5fc3b0e34846d08b06b9d94d5b
Signed-off-by: Yuqi Gu <yu...@arm.com>
---
.../src/common/ambari/do-component-build | 22 +++
.../ambari/patch11-ambari-metrics-mpack.diff | 147 +++++++++++++++++++++
...patch5-refine-AMBARI-25599-for-Hadoop-3.4.diff} | 91 +++++++++----
3 files changed, 232 insertions(+), 28 deletions(-)
diff --git a/bigtop-packages/src/common/ambari/do-component-build b/bigtop-packages/src/common/ambari/do-component-build
index f6e90b88..393e8968 100644
--- a/bigtop-packages/src/common/ambari/do-component-build
+++ b/bigtop-packages/src/common/ambari/do-component-build
@@ -19,6 +19,28 @@ set -ex
export _JAVA_OPTIONS="-Xmx2048m -Djava.awt.headless=true"
+. /etc/os-release
+OS="$ID"
+ARCH=`uname -m`
+RPM_DIR=../../../../../output/ambari
+
mvn clean package -DskipTests -Drat.skip
+# BIGTOP-3800:
+# Build Ambari-mertics RPMs for Mpack deployment;
+# Ambari DO NOT support debian deployment in current version (2.7.5);
+# So just only add x86 RPMs support for Bigtop 3.2+ Mpack (Centos-7).
+if [ $HOSTTYPE = "x86_64" ] ; then
+if [ "${OS}" = "centos" ]; then
+ mkdir -p ${RPM_DIR}/${ARCH}
+ cd ambari-metrics
+ mvn clean package -Dbuild-rpm -DskipTests
+ cd -
+ cp ambari-metrics/ambari-metrics-assembly/target/rpm/ambari-metrics-collector/RPMS/${ARCH}/ambari-metrics-collector*.rpm ${RPM_DIR}/${ARCH}
+ cp ambari-metrics/ambari-metrics-assembly/target/rpm/ambari-metrics-monitor/RPMS/${ARCH}/ambari-metrics-monitor*.rpm ${RPM_DIR}/${ARCH}
+ cp ambari-metrics/ambari-metrics-assembly/target/rpm/ambari-metrics-hadoop-sink/RPMS/${ARCH}/ambari-metrics-hadoop-sink*.rpm ${RPM_DIR}/${ARCH}
+ cp ambari-metrics/ambari-metrics-assembly/target/rpm/ambari-metrics-grafana/RPMS/${ARCH}/ambari-metrics-grafana*.rpm ${RPM_DIR}/${ARCH}
+fi
+fi
+
(cd contrib/management-packs/odpi-ambari-mpack ; mvn clean package -DskipTests -Drat.skip)
diff --git a/bigtop-packages/src/common/ambari/patch11-ambari-metrics-mpack.diff b/bigtop-packages/src/common/ambari/patch11-ambari-metrics-mpack.diff
new file mode 100644
index 00000000..0a82b835
--- /dev/null
+++ b/bigtop-packages/src/common/ambari/patch11-ambari-metrics-mpack.diff
@@ -0,0 +1,147 @@
+diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+index 0a38bdd71f..2cc5c420ad 100644
+--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
++++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+@@ -19,7 +19,7 @@ package org.apache.hadoop.metrics2.sink.timeline;
+
+ import org.apache.commons.configuration2.SubsetConfiguration;
+ import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler;
+-import org.apache.commons.lang.StringUtils;
++import org.apache.commons.lang3.StringUtils;
+ import org.apache.hadoop.classification.InterfaceAudience;
+ import org.apache.hadoop.classification.InterfaceStability;
+ import org.apache.hadoop.metrics2.AbstractMetric;
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
+index cb0e545cf0..7b8277c4d5 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
+@@ -176,10 +176,10 @@ public class ApplicationAttemptFinishDataPBImpl extends
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationAttemptId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
+index 6c76d3cbd9..2e43c209ce 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
+@@ -150,10 +150,10 @@ public class ApplicationAttemptStartDataPBImpl extends
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationAttemptId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
+index ba7e7f0566..2bbf19598b 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationFinishDataPBImpl.java
+@@ -165,10 +165,10 @@ public class ApplicationFinishDataPBImpl extends ApplicationFinishData {
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
+index f47ab00c64..34196f7744 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ApplicationStartDataPBImpl.java
+@@ -189,10 +189,10 @@ public class ApplicationStartDataPBImpl extends ApplicationStartData {
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
+index e9aeb920f9..13fa8c19f1 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerFinishDataPBImpl.java
+@@ -154,10 +154,10 @@ public class ContainerFinishDataPBImpl extends ContainerFinishData {
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.containerId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
+index e7d0762b27..9f4a7e1fbd 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/records/impl/pb/ContainerStartDataPBImpl.java
+@@ -178,10 +178,10 @@ public class ContainerStartDataPBImpl extends ContainerStartData {
+ return false;
+ }
+
+- @Override
+- public String toString() {
+- return TextFormat.shortDebugString(getProto());
+- }
++ //@Override
++ //public String toString() {
++ // return TextFormat.shortDebugString(getProto());
++ //}
+
+ private void mergeLocalToBuilder() {
+ if (this.containerId != null
+diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
+index 51e4b8a61a..1125c28f05 100644
+--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
++++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
+@@ -133,7 +133,7 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
+ @AfterClass
+ public static void doTeardown() throws Exception {
+ dropNonSystemTables();
+- tearDownMiniCluster();
++ tearDownMiniCluster(1);
+ }
+
+ @Before
+@@ -364,4 +364,4 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(query);
+ }
+-}
+\ No newline at end of file
++}
diff --git a/bigtop-packages/src/common/ambari/patch5-AMBARI-25599.diff b/bigtop-packages/src/common/ambari/patch5-refine-AMBARI-25599-for-Hadoop-3.4.diff
similarity index 51%
rename from bigtop-packages/src/common/ambari/patch5-AMBARI-25599.diff
rename to bigtop-packages/src/common/ambari/patch5-refine-AMBARI-25599-for-Hadoop-3.4.diff
index cd730b19..ee520a08 100644
--- a/bigtop-packages/src/common/ambari/patch5-AMBARI-25599.diff
+++ b/bigtop-packages/src/common/ambari/patch5-refine-AMBARI-25599-for-Hadoop-3.4.diff
@@ -1,16 +1,3 @@
-commit 4beb51d38a3361d9d2e7d349b8b3dd203772d7ce
-Author: Szilárd Antal <sa...@cloudera.com>
-Date: Wed Feb 17 11:09:21 2021 +0100
-
- AMBARI-25599 Consider to eliminate HDP public binary references (santal) (#3283)
-
- Change-Id: I7a560a7c2d6ff6c3681fc384d51c23bbb36330c8
- (cherry picked from commit 2e60a76ced3e4ecf1c4356c9727821fe66e8986c)
-
- Conflicts:
- ambari-metrics/ambari-metrics-timelineservice/pom.xml
- ambari-metrics/pom.xml
-
diff --git a/ambari-infra/ambari-infra-assembly/pom.xml b/ambari-infra/ambari-infra-assembly/pom.xml
index 65a630069e..ef4a062960 100644
--- a/ambari-infra/ambari-infra-assembly/pom.xml
@@ -25,25 +12,73 @@ index 65a630069e..ef4a062960 100644
<solr.mapping.path>${mapping.base.path}/ambari-infra-solr</solr.mapping.path>
<solr.package.name>ambari-infra-solr</solr.package.name>
<solr.client.package.name>ambari-infra-solr-client</solr.client.package.name>
+diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
+index 5551b3966f..66d5e8ebc9 100644
+--- a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
++++ b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml
+@@ -199,6 +199,11 @@ limitations under the License.
+ <artifactId>powermock-module-junit4</artifactId>
+ <scope>test</scope>
+ </dependency>
++ <dependency>
++ <groupId>org.apache.commons</groupId>
++ <artifactId>commons-lang3</artifactId>
++ <version>3.12.0</version>
++ </dependency>
+ </dependencies>
+
+ </project>
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
-index f8423b3806..95f9cac688 100644
+index f8423b3806..b20fc15dbb 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
-@@ -34,9 +34,9 @@
+@@ -33,10 +33,10 @@
+ <properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<!--<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>-->
- <protobuf.version>2.5.0</protobuf.version>
+- <protobuf.version>2.5.0</protobuf.version>
- <hadoop.version>3.1.1.3.1.4.0-315</hadoop.version>
- <phoenix.version>5.0.0.3.1.4.0-315</phoenix.version>
- <hbase.version>2.0.2.3.1.4.0-315</hbase.version>
-+ <hadoop.version>3.1.1</hadoop.version>
-+ <phoenix.version>5.0.0-HBase-2.0</phoenix.version>
-+ <hbase.version>2.0.2</hbase.version>
++ <protobuf.version>3.17.3</protobuf.version>
++ <hadoop.version>3.3.4</hadoop.version>
++ <phoenix.version>5.1.2</phoenix.version>
++ <hbase.version>2.4.13</hbase.version>
</properties>
<build>
+@@ -372,9 +372,15 @@
+ </dependency>
+
+ <dependency>
+- <groupId>commons-lang</groupId>
+- <artifactId>commons-lang</artifactId>
+- <version>2.6</version>
++ <groupId>org.apache.phoenix</groupId>
++ <artifactId>phoenix-hbase-compat-2.4.1</artifactId>
++ <version>5.1.2</version>
++ </dependency>
++
++ <dependency>
++ <groupId>org.apache.commons</groupId>
++ <artifactId>commons-lang3</artifactId>
++ <version>3.12.0</version>
+ </dependency>
+
+ <dependency>
+@@ -926,8 +932,8 @@
+ compression="gzip"
+ />
+ <move
+- file="${project.build.directory}/embedded/${phoenix.folder}/phoenix-${phoenix.version}-server.jar"
+- tofile="${project.build.directory}/embedded/${hbase.folder}/lib/phoenix-${phoenix.version}-server.jar"
++ file="${project.build.directory}/embedded/${phoenix.folder}/phoenix-server-hbase-2.4-${phoenix.version}.jar"
++ tofile="${project.build.directory}/embedded/${hbase.folder}/lib/phoenix-server-hbase-2.4-${phoenix.version}.jar"
+ />
+ </target>
+ </configuration>
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
-index a0a11b8e6e..282041fab6 100644
+index a0a11b8e6e..f33479844a 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -40,14 +40,14 @@
@@ -58,14 +93,14 @@ index a0a11b8e6e..282041fab6 100644
- <grafana.tar>https://dl.grafana.com/oss/release/grafana-6.4.2.linux-amd64.tar.gz</grafana.tar>
- <phoenix.tar>https://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/3.x/BUILDS/3.1.4.0-315/tars/phoenix/phoenix-5.0.0.3.1.4.0-315.tar.gz</phoenix.tar>
- <phoenix.folder>phoenix-5.0.0.3.1.4.0-315</phoenix.folder>
-+ <hbase.tar>https://archive.apache.org/dist/hbase/2.0.2/hbase-2.0.2-bin.tar.gz</hbase.tar>
-+ <hbase.folder>hbase-2.0.2</hbase.folder>
-+ <hadoop.tar>https://archive.apache.org/dist/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz</hadoop.tar>
-+ <hadoop.folder>hadoop-3.1.1</hadoop.folder>
-+ <grafana.folder>grafana-6.7.4</grafana.folder>
-+ <grafana.tar>https://dl.grafana.com/oss/release/grafana-6.7.4.linux-amd64.tar.gz</grafana.tar>
-+ <phoenix.tar>https://archive.apache.org/dist/phoenix/apache-phoenix-5.0.0-HBase-2.0/bin/apache-phoenix-5.0.0-HBase-2.0-bin.tar.gz</phoenix.tar>
-+ <phoenix.folder>apache-phoenix-5.0.0-HBase-2.0-bin</phoenix.folder>
++ <hbase.tar>https://archive.apache.org/dist/hbase/2.4.13/hbase-2.4.13-bin.tar.gz</hbase.tar>
++ <hbase.folder>hbase-2.4.13</hbase.folder>
++ <hadoop.tar>https://archive.apache.org/dist/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz</hadoop.tar>
++ <hadoop.folder>hadoop-3.3.4</hadoop.folder>
++ <grafana.folder>grafana-9.1.2</grafana.folder>
++ <grafana.tar>https://dl.grafana.com/enterprise/release/grafana-enterprise-9.1.2.linux-amd64.tar.gz</grafana.tar>
++ <phoenix.tar>https://archive.apache.org/dist/phoenix/phoenix-5.1.2/phoenix-hbase-2.4-5.1.2-bin.tar.gz</phoenix.tar>
++ <phoenix.folder>phoenix-hbase-2.4-5.1.2-bin</phoenix.folder>
<resmonitor.install.dir>/usr/lib/python2.6/site-packages/resource_monitoring</resmonitor.install.dir>
<powermock.version>1.6.2</powermock.version>
<distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>