You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by mk...@apache.org on 2013/11/14 04:00:27 UTC
git commit: CRUNCH-225: Added support for building using Scala 2.10
and 2.9. Also removed unused build.sbt file
Updated Branches:
refs/heads/master a787428ef -> 79e6c896b
CRUNCH-225: Added support for building using Scala 2.10 and 2.9. Also removed unused build.sbt file
Signed-off-by: Micah Whitacre <mk...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/79e6c896
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/79e6c896
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/79e6c896
Branch: refs/heads/master
Commit: 79e6c896bbe0dd5fb57b5358d3f0600657804bb9
Parents: a787428
Author: Micah Whitacre <mk...@apache.org>
Authored: Wed Nov 13 20:22:58 2013 -0600
Committer: Micah Whitacre <mk...@apache.org>
Committed: Wed Nov 13 20:29:03 2013 -0600
----------------------------------------------------------------------
crunch-scrunch/build.sbt | 50 --------------------
crunch-scrunch/pom.xml | 46 +++++++++++++++---
.../org/apache/crunch/scrunch/CogroupTest.scala | 2 +-
.../org/apache/crunch/scrunch/CrunchSuite.scala | 39 +++++++++++++++
.../org/apache/crunch/scrunch/JoinTest.scala | 2 +-
.../apache/crunch/scrunch/PCollectionTest.scala | 2 +-
.../org/apache/crunch/scrunch/PTableTest.scala | 2 +-
.../crunch/scrunch/PageRankClassTest.scala | 2 +-
.../apache/crunch/scrunch/PageRankTest.scala | 2 +-
.../crunch/scrunch/PipelineAppClosureTest.scala | 2 +-
.../apache/crunch/scrunch/PipelineAppTest.scala | 2 +-
.../org/apache/crunch/scrunch/TopTest.scala | 2 +-
.../org/apache/crunch/scrunch/UnionTest.scala | 2 +-
.../apache/crunch/scrunch/WordCountTest.scala | 2 +-
.../interpreter/InterpreterJarTest.scala | 3 +-
.../org/apache/crunch/test/TemporaryPath.java | 8 ++++
pom.xml | 6 ---
17 files changed, 100 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/build.sbt
----------------------------------------------------------------------
diff --git a/crunch-scrunch/build.sbt b/crunch-scrunch/build.sbt
deleted file mode 100644
index 21b52ce..0000000
--- a/crunch-scrunch/build.sbt
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-name := "scrunch"
-
-version := "0.3.0"
-
-scalaVersion := "2.9.2"
-
-resolvers ++= Seq(
- "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository",
-)
-
-libraryDependencies ++= Seq(
- "org.apache.crunch" % "crunch" % "0.3.0" excludeAll(
- ExclusionRule(organization = "com.sun.jdmk"),
- ExclusionRule(organization = "com.sun.jmx"),
- ExclusionRule(organization = "javax.jms")
- ),
- "org.apache.hadoop" % "hadoop-client" % "1.0.3" % "provided" excludeAll(
- ExclusionRule(organization = "com.sun.jdmk"),
- ExclusionRule(organization = "com.sun.jmx"),
- ExclusionRule(organization = "javax.jms")
- ),
- "org.apache.hbase" % "hbase" % "0.90.4" % "provided" excludeAll(
- ExclusionRule(organization = "org.apache.hadoop"),
- ExclusionRule(organization = "commons-logging"),
- ExclusionRule(organization = "com.google.guava"),
- ExclusionRule(organization = "log4j"),
- ExclusionRule(organization = "org.slf4j")
- ),
- "junit" % "junit" % "4.8.1" % "test",
- "org.scalatest" % "scalatest_2.9.2" % "1.7.2" % "test"
-)
-
-parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml
index 10c73c9..b6d7b54 100644
--- a/crunch-scrunch/pom.xml
+++ b/crunch-scrunch/pom.xml
@@ -60,11 +60,6 @@ under the License.
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.version}</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-test</artifactId>
<scope>test</scope>
@@ -209,6 +204,45 @@ under the License.
</plugins>
</build>
</profile>
+ <profile>
+ <id>scala-2.9.2</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <scala.version>2.9.2</scala.version>
+ <scalatest.version>1.7.2</scalatest.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.version}</artifactId>
+ </dependency>
+ </dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.version}</artifactId>
+ <version>${scalatest.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+ <profile>
+ <id>scala-2.10</id>
+ <properties>
+ <scala.version>2.10.2</scala.version>
+ <scalatest.version>1.9.2</scalatest.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_2.10</artifactId>
+ <version>${scalatest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
-
</project>
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
index 170b8c9..40df05b 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
import _root_.org.junit.Test
-class CogroupTest extends CrunchTestSupport with JUnitSuite {
+class CogroupTest extends CrunchSuite {
lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
def wordCount(fileName: String) = {
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
new file mode 100644
index 0000000..37a4fe2
--- /dev/null
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.{After, Before, Rule}
+import org.apache.crunch.test.TemporaryPath
+
+class CrunchSuite extends JUnitSuite{
+
+ val tempDir = new TemporaryPath("crunch.tmp.dir", "hadoop.tmp.dir");
+
+ def getFolder() = {
+ tempDir
+ }
+
+ @Before def initialize() {
+ tempDir.create()
+ }
+
+ @After def cleanup() {
+ tempDir.delete()
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
index 0770ba3..bd640eb 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
import _root_.org.junit.Test
-class JoinTest extends CrunchTestSupport with JUnitSuite {
+class JoinTest extends CrunchSuite {
lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
def wordCount(fileName: String) = {
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
index 94ac917..6d42ed6 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
@@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite
/**
* Tests functionality of Scala PCollection.
*/
-class PCollectionTest extends CrunchTestSupport with JUnitSuite {
+class PCollectionTest extends CrunchSuite {
// Number of lines in the Shakespeare data set.
val linesInShakespeare: Int = 3667
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
index cd644b1..35b2a8d 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
@@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite
/**
* Tests functionality of Scala PTable.
*/
-class PTableTest extends CrunchTestSupport with JUnitSuite {
+class PTableTest extends CrunchSuite {
/**
* Gets a PCollection containing the lines from the tens data set.
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
index 847d65b..55e7783 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
@@ -61,7 +61,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo
}
}
-class PageRankClassTest extends CrunchTestSupport with JUnitSuite {
+class PageRankClassTest extends CrunchSuite {
lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
index 02e3d82..833fce8 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
@@ -49,7 +49,7 @@ class CachingPageRankFn extends DoFn[P[String, (Float, Float, List[String])], P[
}
}
-class PageRankTest extends CrunchTestSupport with JUnitSuite {
+class PageRankTest extends CrunchSuite {
lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
def initialInput(fileName: String) = {
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
index e0702e4..4666c3c 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
@@ -27,7 +27,7 @@ import _root_.org.junit.Test
* Test that verifies that a Scala PipelineApp can properly send some side data as part of a
* function closure.
*/
-class PipelineClosureAppTest extends CrunchTestSupport with JUnitSuite {
+class PipelineClosureAppTest extends CrunchSuite {
/**
* A simple pipeline application that divides each element of a PCollection of numbers by
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
index db49c92..fffd529 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
@@ -39,7 +39,7 @@ object WordCount extends PipelineApp {
}
}
-class PipelineAppTest extends CrunchTestSupport with JUnitSuite {
+class PipelineAppTest extends CrunchSuite {
@Test def run {
val args = new Array[String](3)
args(0) = tempDir.copyResourceFileName("shakes.txt")
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
index 0c7a992..ca728e8 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
import _root_.org.junit.Test
-class TopTest extends CrunchTestSupport with JUnitSuite {
+class TopTest extends CrunchSuite {
@Test def topInMem {
val ptable = Mem.tableOf(("foo", 17), ("bar", 29), ("baz", 1729))
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
index e8b3fab..a507e37 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
import _root_.org.junit.Test
-class UnionTest extends CrunchTestSupport with JUnitSuite {
+class UnionTest extends CrunchSuite {
lazy val pipeline = Pipeline.mapReduce[UnionTest](tempDir.getDefaultConfiguration)
def wordCount(col: PCollection[String]) = {
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
index a2560d1..2cc1457 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
import _root_.org.junit.Test
-class WordCountTest extends CrunchTestSupport with JUnitSuite {
+class WordCountTest extends CrunchSuite {
@Test def wordCount {
val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration)
val input = tempDir.copyResourceFileName("shakes.txt")
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
index 48aae00..5ebc303 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
@@ -29,11 +29,12 @@ import org.junit.Assert.assertNotNull
import org.junit.Test
import org.apache.crunch.test.CrunchTestSupport
import org.scalatest.junit.JUnitSuite
+import org.apache.crunch.scrunch.CrunchSuite
/**
* Tests creating jars from a {@link scala.tools.nsc.io.VirtualDirectory}.
*/
-class InterpreterJarTest extends CrunchTestSupport with JUnitSuite {
+class InterpreterJarTest extends CrunchSuite {
/**
* Tests transforming a virtual directory into a temporary jar file.
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
index 3cd175f..1d25fc7 100644
--- a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
+++ b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
@@ -59,6 +59,14 @@ public final class TemporaryPath extends ExternalResource {
}
}
+ public void create() throws Throwable {
+ tmp.create();
+ }
+
+ public void delete() {
+ tmp.delete();
+ }
+
@Override
public Statement apply(Statement base, Description description) {
return tmp.apply(base, description);
http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 338b4f3..7e288d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -352,12 +352,6 @@ under the License.
<artifactId>jline</artifactId>
<version>${scala.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.version}</artifactId>
- <version>${scalatest.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>