You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ju...@apache.org on 2018/09/19 19:11:43 UTC
[2/2] bigtop git commit: BIGTOP-3082: Fix build failure with
flume-1.8+kafka-0.10.2.2
BIGTOP-3082: Fix build failure with flume-1.8+kafka-0.10.2.2
When upgrading flume to 1.8 and kafka to 0.10.2.2, the build of flume
is failed due to API changes and scala upgrade in kafka.
This patch rebased FLUME-2662/FLUME-3026 patches to flume-1.8, and for
the scala conflicts, used Anton's patch which can be found here:
https://lists.apache.org/thread.html/%3CCAJZrK_tN49GM+vCCiB+rjLL==82U=7nswoapwWmyJbVAS5F=QA@mail.gmail.com%3E
This closes #397
Change-Id: Idf8d00c22ef60d82783056fea556e9a6f517054a
Signed-off-by: Jun He <ju...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fc97b9ee
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fc97b9ee
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fc97b9ee
Branch: refs/heads/master
Commit: fc97b9ee767222fa95ba012677637fb4569a6695
Parents: 2135145
Author: Jun He <ju...@linaro.org>
Authored: Thu Sep 20 01:48:25 2018 +0800
Committer: Jun He <ju...@apache.org>
Committed: Wed Sep 19 19:10:10 2018 +0000
----------------------------------------------------------------------
.../src/common/flume/patch0-FLUME-2662.diff | 24 ++---
.../src/common/flume/patch1-FLUME-3026.diff | 92 ------------------
.../common/flume/patch1-FLUME-3026_rebased.diff | 99 ++++++++++++++++++++
bigtop.bom | 4 +-
4 files changed, 114 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc97b9ee/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff b/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff
index 9644c5e..61497b9 100644
--- a/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff
+++ b/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff
@@ -1,11 +1,13 @@
---- apache-flume-1.7.0-src/pom.xml. 2016-11-19 20:56:05.683682127 +0100
-+++ apache-flume-1.7.0-src/pom.xml 2016-11-19 20:56:22.707682094 +0100
-@@ -951,7 +951,7 @@
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
-- <version>2.1</version>
-+ <version>2.4</version>
- </dependency>
-
- <dependency>
+diff --git a/pom.xml b/pom.xml
+index 3c82a47..bdd998d 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -58,7 +58,7 @@ limitations under the License.
+ <commons-collections.version>3.2.2</commons-collections.version>
+ <commons-compress.version>1.4.1</commons-compress.version>
+ <commons-dbcp.version>1.4</commons-dbcp.version>
+- <commons-io.version>2.1</commons-io.version>
++ <commons-io.version>2.4</commons-io.version>
+ <commons-lang.version>2.5</commons-lang.version>
+ <curator.version>2.6.0</curator.version>
+ <derby.version>10.11.1.1</derby.version>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc97b9ee/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff b/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff
deleted file mode 100644
index 503de9d..0000000
--- a/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff
+++ /dev/null
@@ -1,92 +0,0 @@
-diff --git a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
-index 5e5f2d0..917cee2 100644
---- a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
-+++ b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
-@@ -20,6 +20,7 @@ package org.apache.flume.channel.kafka;
-
- import com.google.common.collect.Lists;
- import kafka.admin.AdminUtils;
-+import kafka.admin.RackAwareMode;
- import kafka.utils.ZKGroupTopicDirs;
- import kafka.utils.ZkUtils;
- import org.apache.commons.lang.RandomStringUtils;
-@@ -883,7 +884,8 @@ public class TestKafkaChannel {
- ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false);
- int replicationFactor = 1;
- Properties topicConfig = new Properties();
-- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
-+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
-+ RackAwareMode.Disabled$.MODULE$);
- }
-
- public static void deleteTopic(String topicName) {
-diff --git a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
-index 7c66420..bc2a299 100644
---- a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
-+++ b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
-@@ -21,6 +21,7 @@ package org.apache.flume.sink.kafka;
- import com.google.common.base.Charsets;
-
- import kafka.admin.AdminUtils;
-+import kafka.admin.RackAwareMode;
- import kafka.message.MessageAndMetadata;
- import kafka.utils.ZkUtils;
-
-@@ -551,7 +552,8 @@ public class TestKafkaSink {
- ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false);
- int replicationFactor = 1;
- Properties topicConfig = new Properties();
-- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
-+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
-+ RackAwareMode.Disabled$.MODULE$);
- }
-
- public static void deleteTopic(String topicName) {
-diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
-index 53bd65c..ae5348c 100644
---- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
-+++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
-@@ -17,6 +17,7 @@
- package org.apache.flume.source.kafka;
-
- import kafka.admin.AdminUtils;
-+import kafka.admin.RackAwareMode;
- import kafka.server.KafkaConfig;
- import kafka.server.KafkaServerStartable;
- import kafka.utils.ZkUtils;
-@@ -131,7 +132,8 @@ public class KafkaSourceEmbeddedKafka {
- ZkUtils zkUtils = ZkUtils.apply(zkClient, false);
- int replicationFactor = 1;
- Properties topicConfig = new Properties();
-- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
-+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
-+ RackAwareMode.Disabled$.MODULE$);
- }
-
- }
-diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
-index d1daceb..cda91f9 100644
---- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
-+++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
-@@ -20,7 +20,7 @@ package org.apache.flume.source.kafka;
- import com.google.common.base.Charsets;
- import com.google.common.collect.Lists;
- import junit.framework.Assert;
--import kafka.common.TopicExistsException;
-+import org.apache.kafka.common.errors.TopicExistsException;
- import kafka.utils.ZKGroupTopicDirs;
- import kafka.utils.ZkUtils;
- import org.apache.avro.io.BinaryEncoder;
-diff --git a/pom.xml b/pom.xml
-index f62c99a..fb2340f 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -52,7 +52,7 @@ limitations under the License.
- <elasticsearch.version>0.90.1</elasticsearch.version>
- <hadoop2.version>2.4.0</hadoop2.version>
- <thrift.version>0.7.0</thrift.version>
-- <kafka.version>0.9.0.1</kafka.version>
-+ <kafka.version>0.10.1.0</kafka.version>
- <kite.version>1.0.0</kite.version>
- <hive.version>1.0.0</hive.version>
- <xalan.verion>2.7.1</xalan.verion>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc97b9ee/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff b/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff
new file mode 100644
index 0000000..20a2ddd
--- /dev/null
+++ b/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff
@@ -0,0 +1,99 @@
+diff --git a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
+index 5e5f2d0..63607f7 100644
+--- a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
++++ b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java
+@@ -20,6 +20,7 @@ package org.apache.flume.channel.kafka;
+
+ import com.google.common.collect.Lists;
+ import kafka.admin.AdminUtils;
++import kafka.admin.RackAwareMode;
+ import kafka.utils.ZKGroupTopicDirs;
+ import kafka.utils.ZkUtils;
+ import org.apache.commons.lang.RandomStringUtils;
+@@ -883,7 +884,8 @@ public class TestKafkaChannel {
+ ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false);
+ int replicationFactor = 1;
+ Properties topicConfig = new Properties();
+- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
++ RackAwareMode.Disabled$.MODULE$);
+ }
+
+ public static void deleteTopic(String topicName) {
+diff --git a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
+index d92c71f..66c6fe3 100644
+--- a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
++++ b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java
+@@ -21,6 +21,7 @@ package org.apache.flume.sink.kafka;
+ import com.google.common.base.Charsets;
+
+ import kafka.admin.AdminUtils;
++import kafka.admin.RackAwareMode;
+ import kafka.message.MessageAndMetadata;
+ import kafka.utils.ZkUtils;
+
+@@ -674,7 +675,8 @@ public class TestKafkaSink {
+ ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false);
+ int replicationFactor = 1;
+ Properties topicConfig = new Properties();
+- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
++ RackAwareMode.Disabled$.MODULE$);
+ }
+
+ public static void deleteTopic(String topicName) {
+@@ -698,4 +700,4 @@ public class TestKafkaSink {
+ return newTopic;
+ }
+
+-}
+\ No newline at end of file
++}
+diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
+index 53bd65c..ba75623 100644
+--- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
++++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java
+@@ -17,6 +17,7 @@
+ package org.apache.flume.source.kafka;
+
+ import kafka.admin.AdminUtils;
++import kafka.admin.RackAwareMode;
+ import kafka.server.KafkaConfig;
+ import kafka.server.KafkaServerStartable;
+ import kafka.utils.ZkUtils;
+@@ -131,7 +132,8 @@ public class KafkaSourceEmbeddedKafka {
+ ZkUtils zkUtils = ZkUtils.apply(zkClient, false);
+ int replicationFactor = 1;
+ Properties topicConfig = new Properties();
+- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig);
++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig,
++ RackAwareMode.Disabled$.MODULE$);
+ }
+
+ }
+diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
+index 7804fa2..2d5bbf8 100644
+--- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
++++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java
+@@ -20,7 +20,7 @@ package org.apache.flume.source.kafka;
+ import com.google.common.base.Charsets;
+ import com.google.common.collect.Lists;
+ import junit.framework.Assert;
+-import kafka.common.TopicExistsException;
++import org.apache.kafka.common.errors.TopicExistsException;
+ import kafka.utils.ZKGroupTopicDirs;
+ import kafka.utils.ZkUtils;
+ import org.apache.avro.io.BinaryEncoder;
+diff --git a/pom.xml b/pom.xml
+index 3c82a47..2276355 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -77,7 +77,7 @@ limitations under the License.
+ <jetty.version>6.1.26</jetty.version>
+ <joda-time.version>2.9.9</joda-time.version>
+ <junit.version>4.10</junit.version>
+- <kafka.version>0.9.0.1</kafka.version>
++ <kafka.version>0.10.2.2</kafka.version>
+ <kite.version>1.0.0</kite.version>
+ <hive.version>1.0.0</hive.version>
+ <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc97b9ee/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 0543978..02e5b96 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -235,7 +235,7 @@ bigtop {
'flume' {
name = 'flume'
relNotes = 'Apache Flume'
- version { base = '1.7.0'; pkg = base; release = 1 }
+ version { base = '1.8.0'; pkg = base; release = 1 }
tarball { destination = "apache-$name-${version.base}-src.tar.gz"
source = destination }
url { download_path = "/$name/${version.base}/"
@@ -383,7 +383,7 @@ bigtop {
'kafka' {
name = 'kafka'
relNotes = 'Apache Kafka'
- version { base = '0.10.1.1'; pkg = base; release = 1 }
+ version { base = '0.10.2.2'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}-src.tgz" }
url { download_path = "/$name/${version.base}/"