You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/07/02 22:49:50 UTC
spark git commit: [SPARK-8781] Fix variables in published pom.xml are
not resolved
Repository: spark
Updated Branches:
refs/heads/master 34d448dbe -> 82cf3315e
[SPARK-8781] Fix variables in published pom.xml are not resolved
The issue is summarized in the JIRA and is caused by this commit: 984ad60147c933f2d5a2040c87ae687c14eb1724.
This patch reverts that commit and fixes the maven build in a different way. We limit the dependencies of `KinesisReceiverSuite` to avoid having to deal with the complexities in how maven deals with transitive test dependencies.
Author: Andrew Or <an...@databricks.com>
Closes #7193 from andrewor14/fix-kinesis-pom and squashes the following commits:
ca3d5d4 [Andrew Or] Limit kinesis test dependencies
f24e09c [Andrew Or] Revert "[BUILD] Fix Maven build for Kinesis"
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/82cf3315
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/82cf3315
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/82cf3315
Branch: refs/heads/master
Commit: 82cf3315e690f4ac15b50edea6a3d673aa5be4c0
Parents: 34d448d
Author: Andrew Or <an...@databricks.com>
Authored: Thu Jul 2 13:49:45 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Jul 2 13:49:45 2015 -0700
----------------------------------------------------------------------
extras/kinesis-asl/pom.xml | 7 -------
.../kinesis/KinesisReceiverSuite.scala | 20 ++++++++++++--------
pom.xml | 2 --
3 files changed, 12 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/82cf3315/extras/kinesis-asl/pom.xml
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml
index c242e7a..5289073 100644
--- a/extras/kinesis-asl/pom.xml
+++ b/extras/kinesis-asl/pom.xml
@@ -42,13 +42,6 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/spark/blob/82cf3315/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
index 2103dca..6c26262 100644
--- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
+++ b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
@@ -26,18 +26,23 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn
import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason
import com.amazonaws.services.kinesis.model.Record
import org.mockito.Mockito._
-import org.scalatest.{BeforeAndAfter, Matchers}
+// scalastyle:off
+// To avoid introducing a dependency on Spark core tests, simply use scalatest's FunSuite
+// here instead of our own SparkFunSuite. Introducing the dependency has caused problems
+// in the past (SPARK-8781) that are complicated by bugs in the maven shade plugin (MSHADE-148).
+import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
import org.scalatest.mock.MockitoSugar
import org.apache.spark.storage.StorageLevel
-import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext, TestSuiteBase}
+import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext}
import org.apache.spark.util.{Clock, ManualClock, Utils}
/**
* Suite of Kinesis streaming receiver tests focusing mostly on the KinesisRecordProcessor
*/
-class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAfter
- with MockitoSugar {
+class KinesisReceiverSuite extends FunSuite with Matchers with BeforeAndAfter
+ with MockitoSugar {
+// scalastyle:on
val app = "TestKinesisReceiver"
val stream = "mySparkStream"
@@ -57,7 +62,7 @@ class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAft
var checkpointStateMock: KinesisCheckpointState = _
var currentClockMock: Clock = _
- override def beforeFunction(): Unit = {
+ before {
receiverMock = mock[KinesisReceiver]
checkpointerMock = mock[IRecordProcessorCheckpointer]
checkpointClockMock = mock[ManualClock]
@@ -65,8 +70,7 @@ class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAft
currentClockMock = mock[Clock]
}
- override def afterFunction(): Unit = {
- super.afterFunction()
+ after {
// Since this suite was originally written using EasyMock, add this to preserve the old
// mocking semantics (see SPARK-5735 for more details)
verifyNoMoreInteractions(receiverMock, checkpointerMock, checkpointClockMock,
@@ -74,7 +78,7 @@ class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAft
}
test("KinesisUtils API") {
- val ssc = new StreamingContext(master, framework, batchDuration)
+ val ssc = new StreamingContext("local[2]", getClass.getSimpleName, Seconds(1))
// Tests the API, does not actually test data receiving
val kinesisStream1 = KinesisUtils.createStream(ssc, "mySparkStream",
"https://kinesis.us-west-2.amazonaws.com", Seconds(2),
http://git-wip-us.apache.org/repos/asf/spark/blob/82cf3315/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 211da9e..ffa9612 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1440,8 +1440,6 @@
<version>2.3</version>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
- <!-- Work around MSHADE-148 -->
- <createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<includes>
<!-- At a minimum we must include this to force effective pom generation -->
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org