You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/09/29 18:46:25 UTC
spark git commit: Revert "[SPARK-22142][BUILD][STREAMING] Move Flume
support behind a profile"
Repository: spark
Updated Branches:
refs/heads/master 9ed7394a6 -> 472864014
Revert "[SPARK-22142][BUILD][STREAMING] Move Flume support behind a profile"
This reverts commit a2516f41aef68e39df7f6380fd2618cc148a609e.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/47286401
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/47286401
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/47286401
Branch: refs/heads/master
Commit: 472864014c42da08b9d3f3fffbe657c6fcf1e2ef
Parents: 9ed7394
Author: gatorsmile <ga...@gmail.com>
Authored: Fri Sep 29 11:45:58 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Fri Sep 29 11:45:58 2017 -0700
----------------------------------------------------------------------
dev/create-release/release-build.sh | 4 ++--
dev/mima | 2 +-
dev/scalastyle | 1 -
dev/sparktestsupport/modules.py | 20 +-------------------
dev/test-dependencies.sh | 2 +-
docs/building-spark.md | 6 ------
pom.xml | 13 +++----------
project/SparkBuild.scala | 17 ++++++++---------
python/pyspark/streaming/tests.py | 16 +++-------------
9 files changed, 19 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/dev/create-release/release-build.sh
----------------------------------------------------------------------
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 7e8d5c7..5390f59 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -84,9 +84,9 @@ MVN="build/mvn --force"
# Hive-specific profiles for some builds
HIVE_PROFILES="-Phive -Phive-thriftserver"
# Profiles for publishing snapshots and release to Maven Central
-PUBLISH_PROFILES="-Pmesos -Pyarn -Pflume $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
+PUBLISH_PROFILES="-Pmesos -Pyarn $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
# Profiles for building binary releases
-BASE_RELEASE_PROFILES="-Pmesos -Pyarn -Pflume -Psparkr"
+BASE_RELEASE_PROFILES="-Pmesos -Pyarn -Psparkr"
# Scala 2.11 only profiles for some builds
SCALA_2_11_PROFILES="-Pkafka-0-8"
# Scala 2.12 only profiles for some builds
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/dev/mima
----------------------------------------------------------------------
diff --git a/dev/mima b/dev/mima
index 1e3ca97..fdb21f5 100755
--- a/dev/mima
+++ b/dev/mima
@@ -24,7 +24,7 @@ set -e
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
-SPARK_PROFILES="-Pmesos -Pkafka-0-8 -Pyarn -Pflume -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
+SPARK_PROFILES="-Pmesos -Pkafka-0-8 -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/dev/scalastyle
----------------------------------------------------------------------
diff --git a/dev/scalastyle b/dev/scalastyle
index 89ecc8a..e5aa589 100755
--- a/dev/scalastyle
+++ b/dev/scalastyle
@@ -25,7 +25,6 @@ ERRORS=$(echo -e "q\n" \
-Pmesos \
-Pkafka-0-8 \
-Pyarn \
- -Pflume \
-Phive \
-Phive-thriftserver \
scalastyle test:scalastyle \
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/dev/sparktestsupport/modules.py
----------------------------------------------------------------------
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 91d5667..50e14b6 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -279,12 +279,6 @@ streaming_flume_sink = Module(
source_file_regexes=[
"external/flume-sink",
],
- build_profile_flags=[
- "-Pflume",
- ],
- environ={
- "ENABLE_FLUME_TESTS": "1"
- },
sbt_test_goals=[
"streaming-flume-sink/test",
]
@@ -297,12 +291,6 @@ streaming_flume = Module(
source_file_regexes=[
"external/flume",
],
- build_profile_flags=[
- "-Pflume",
- ],
- environ={
- "ENABLE_FLUME_TESTS": "1"
- },
sbt_test_goals=[
"streaming-flume/test",
]
@@ -314,13 +302,7 @@ streaming_flume_assembly = Module(
dependencies=[streaming_flume, streaming_flume_sink],
source_file_regexes=[
"external/flume-assembly",
- ],
- build_profile_flags=[
- "-Pflume",
- ],
- environ={
- "ENABLE_FLUME_TESTS": "1"
- }
+ ]
)
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/dev/test-dependencies.sh
----------------------------------------------------------------------
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index 58b295d..c771457 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -29,7 +29,7 @@ export LC_ALL=C
# TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
# NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkafka-0-8 -Pyarn -Pflume -Phive"
+HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkafka-0-8 -Pyarn -Phive"
MVN="build/mvn"
HADOOP_PROFILES=(
hadoop-2.6
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/docs/building-spark.md
----------------------------------------------------------------------
diff --git a/docs/building-spark.md b/docs/building-spark.md
index e1532de..57baa50 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -100,12 +100,6 @@ Note: Kafka 0.8 support is deprecated as of Spark 2.3.0.
Kafka 0.10 support is still automatically built.
-## Building with Flume support
-
-Apache Flume support must be explicitly enabled with the `flume` profile.
-
- ./build/mvn -Pflume -DskipTests clean package
-
## Building submodules individually
It's possible to build Spark sub-modules using the `mvn -pl` option.
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9fac8b1..87a468c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,13 +98,15 @@
<module>sql/core</module>
<module>sql/hive</module>
<module>assembly</module>
+ <module>external/flume</module>
+ <module>external/flume-sink</module>
+ <module>external/flume-assembly</module>
<module>examples</module>
<module>repl</module>
<module>launcher</module>
<module>external/kafka-0-10</module>
<module>external/kafka-0-10-assembly</module>
<module>external/kafka-0-10-sql</module>
- <!-- See additional modules enabled by profiles below -->
</modules>
<properties>
@@ -2581,15 +2583,6 @@
</dependencies>
</profile>
- <profile>
- <id>flume</id>
- <modules>
- <module>external/flume</module>
- <module>external/flume-sink</module>
- <module>external/flume-assembly</module>
- </modules>
- </profile>
-
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
<profile>
<id>spark-ganglia-lgpl</id>
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 9501eed..a568d26 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -43,8 +43,11 @@ object BuildCommons {
"catalyst", "sql", "hive", "hive-thriftserver", "sql-kafka-0-10"
).map(ProjectRef(buildLocation, _))
- val streamingProjects@Seq(streaming, streamingKafka010) =
- Seq("streaming", "streaming-kafka-0-10").map(ProjectRef(buildLocation, _))
+ val streamingProjects@Seq(
+ streaming, streamingFlumeSink, streamingFlume, streamingKafka010
+ ) = Seq(
+ "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka-0-10"
+ ).map(ProjectRef(buildLocation, _))
val allProjects@Seq(
core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _*
@@ -53,13 +56,9 @@ object BuildCommons {
"tags", "sketch", "kvstore"
).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects
- val optionallyEnabledProjects@Seq(mesos, yarn,
- streamingFlumeSink, streamingFlume,
- streamingKafka, sparkGangliaLgpl, streamingKinesisAsl,
- dockerIntegrationTests, hadoopCloud) =
- Seq("mesos", "yarn",
- "streaming-flume-sink", "streaming-flume",
- "streaming-kafka-0-8", "ganglia-lgpl", "streaming-kinesis-asl",
+ val optionallyEnabledProjects@Seq(mesos, yarn, streamingKafka, sparkGangliaLgpl,
+ streamingKinesisAsl, dockerIntegrationTests, hadoopCloud) =
+ Seq("mesos", "yarn", "streaming-kafka-0-8", "ganglia-lgpl", "streaming-kinesis-asl",
"docker-integration-tests", "hadoop-cloud").map(ProjectRef(buildLocation, _))
val assemblyProjects@Seq(networkYarn, streamingFlumeAssembly, streamingKafkaAssembly, streamingKafka010Assembly, streamingKinesisAslAssembly) =
http://git-wip-us.apache.org/repos/asf/spark/blob/47286401/python/pyspark/streaming/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py
index 5b86c1c..229cf53 100644
--- a/python/pyspark/streaming/tests.py
+++ b/python/pyspark/streaming/tests.py
@@ -1478,7 +1478,7 @@ def search_kafka_assembly_jar():
("Failed to find Spark Streaming kafka assembly jar in %s. " % kafka_assembly_dir) +
"You need to build Spark with "
"'build/sbt assembly/package streaming-kafka-0-8-assembly/assembly' or "
- "'build/mvn -Pkafka-0-8 package' before running this test.")
+ "'build/mvn package' before running this test.")
elif len(jars) > 1:
raise Exception(("Found multiple Spark Streaming Kafka assembly JARs: %s; please "
"remove all but one") % (", ".join(jars)))
@@ -1495,7 +1495,7 @@ def search_flume_assembly_jar():
("Failed to find Spark Streaming Flume assembly jar in %s. " % flume_assembly_dir) +
"You need to build Spark with "
"'build/sbt assembly/assembly streaming-flume-assembly/assembly' or "
- "'build/mvn -Pflume package' before running this test.")
+ "'build/mvn package' before running this test.")
elif len(jars) > 1:
raise Exception(("Found multiple Spark Streaming Flume assembly JARs: %s; please "
"remove all but one") % (", ".join(jars)))
@@ -1517,9 +1517,6 @@ def search_kinesis_asl_assembly_jar():
# Must be same as the variable and condition defined in modules.py
-flume_test_environ_var = "ENABLE_FLUME_TESTS"
-are_flume_tests_enabled = os.environ.get(flume_test_environ_var) == '1'
-# Must be same as the variable and condition defined in modules.py
kafka_test_environ_var = "ENABLE_KAFKA_0_8_TESTS"
are_kafka_tests_enabled = os.environ.get(kafka_test_environ_var) == '1'
# Must be same as the variable and condition defined in KinesisTestUtils.scala and modules.py
@@ -1541,16 +1538,9 @@ if __name__ == "__main__":
os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars %s pyspark-shell" % jars
testcases = [BasicOperationTests, WindowFunctionTests, StreamingContextTests, CheckpointTests,
+ FlumeStreamTests, FlumePollingStreamTests,
StreamingListenerTests]
- if are_flume_tests_enabled:
- testcases.append(FlumeStreamTests)
- testcases.append(FlumePollingStreamTests)
- else:
- sys.stderr.write(
- "Skipped test_flume_stream (enable by setting environment variable %s=1"
- % flume_test_environ_var)
-
if are_kafka_tests_enabled:
testcases.append(KafkaStreamTests)
else:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org