You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2012/08/22 08:26:48 UTC
[2/9] git commit: CRUNCH-16: Fix dependency versions.
CRUNCH-16: Fix dependency versions.
Move all dependency versions to properties for easier management.
Align Crunch's dependencies with those on Hadoop's classpath.
Remove Scrunch's last non-test dependency on commons-io.
Add HBase warning note to README.
Signed-off-by: Josh Wills <jw...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/9d8b679e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/9d8b679e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/9d8b679e
Branch: refs/heads/master
Commit: 9d8b679ec0bbd007e65c33f2018b2c9630cfc700
Parents: 5d9b698
Author: Matthias Friedrich <ma...@mafr.de>
Authored: Thu Aug 16 20:47:19 2012 +0200
Committer: Josh Wills <jw...@cloudera.com>
Committed: Tue Aug 21 19:46:28 2012 -0700
----------------------------------------------------------------------
README | 6 ++
crunch-hbase/pom.xml | 7 ++
.../scrunch/interpreter/InterpreterRunner.scala | 4 +-
crunch-test/pom.xml | 6 ++
crunch/pom.xml | 11 ++-
pom.xml | 59 ++++++++++-----
6 files changed, 68 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/README
----------------------------------------------------------------------
diff --git a/README b/README
index d061cab..b80235d 100644
--- a/README
+++ b/README
@@ -24,3 +24,9 @@ local Maven cache:
mvn install
+Crunch has experimental support for Hadoop 2 through the "hadoop-2" build
+profile (add -Phadoop-2 to enable it). If you want to use HBase support on
+Hadoop 2, please note that you have to build HBase 0.94.1 from source using
+the following command:
+
+ mvn clean install -Dhadoop.profile=2.0
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-hbase/pom.xml b/crunch-hbase/pom.xml
index 3bd6e54..d97b457 100644
--- a/crunch-hbase/pom.xml
+++ b/crunch-hbase/pom.xml
@@ -68,6 +68,13 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
index 18cb201..196e222 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
@@ -33,7 +33,7 @@ import scala.tools.nsc.io.Jar
import scala.tools.nsc.io.VirtualDirectory
import com.google.common.io.Files
-import org.apache.commons.io.IOUtils
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.Configuration
import org.apache.crunch.util.DistCache
@@ -153,7 +153,7 @@ object InterpreterRunner extends MainGenericRunner {
addVirtualDirectoryToJar(virtualDirectory, "", jarStream)
return tempJar
} finally {
- IOUtils.closeQuietly(jarStream)
+ Closeables.closeQuietly(jarStream)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-test/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index 8a2f019..a1e3899 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -48,6 +48,12 @@ under the License.
<artifactId>hadoop-minicluster</artifactId>
</dependency>
+ <!-- Required by LocalJobRunner -->
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch/pom.xml b/crunch/pom.xml
index 20c75b7..07951ec 100644
--- a/crunch/pom.xml
+++ b/crunch/pom.xml
@@ -52,23 +52,21 @@ under the License.
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Both Protobufs and Thrift are supported as
@@ -109,16 +107,19 @@ under the License.
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e397663..1c5b4c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,8 +57,28 @@ under the License.
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.source.version>1.6</java.source.version>
<java.target.version>1.6</java.target.version>
- <scala.version>2.9.2</scala.version>
+ <!--
+ NOTE: Please don't change any dependency versions here without
+ checking first that they match those on Hadoop's runtime
+ classpath.
+ -->
+ <guava.version>11.0.2</guava.version>
+ <commons-io.version>2.1</commons-io.version>
+ <commons-lang.version>2.4</commons-lang.version>
+ <commons-codec.version>1.4</commons-codec.version>
+ <commons-httpclient.version>3.0.1</commons-httpclient.version>
+ <commons-logging.version>1.1.1</commons-logging.version>
<avro.version>1.7.0</avro.version>
+ <jackson.version>1.8.8</jackson.version>
+ <protobuf-java.version>2.3.0</protobuf-java.version>
+ <libthrift.version>0.8.0</libthrift.version>
+ <slf4j.version>1.4.3</slf4j.version>
+ <log4j.version>1.2.15</log4j.version>
+ <junit.version>4.10</junit.version>
+ <hamcrest.version>1.1</hamcrest.version>
+ <mockito.version>1.9.0</mockito.version>
+ <scala.version>2.9.2</scala.version>
+ <scalatest.version>1.7.2</scalatest.version>
</properties>
<scm>
@@ -119,7 +139,7 @@ under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>11.0.2</version>
+ <version>${guava.version}</version>
</dependency>
<dependency>
@@ -137,79 +157,79 @@ under the License.
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.2</version>
+ <version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.4</version>
+ <version>${commons-codec.version}</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
- <version>2.4</version>
+ <version>${commons-lang.version}</version>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
- <version>3.0.1</version>
+ <version>${commons-httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
- <version>1.8.3</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
- <version>1.8.3</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>2.4.1</version>
+ <version>${protobuf-java.version}</version>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.8.0</version>
+ <version>${libthrift.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.10</version>
+ <version>${junit.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
- <version>1.9.0</version>
+ <version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
- <version>1.1</version>
+ <version>${hamcrest.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <version>1.1.1</version>
+ <version>${commons-logging.version}</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>1.2.15</version>
+ <version>${log4j.version}</version>
<exclusions>
<exclusion>
<groupId>javax.mail</groupId>
@@ -233,13 +253,13 @@ under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.4.3</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>1.4.3</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
@@ -263,7 +283,7 @@ under the License.
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.version}</artifactId>
- <version>1.7.2</version>
+ <version>${scalatest.version}</version>
</dependency>
</dependencies>
@@ -357,6 +377,9 @@ under the License.
mvn clean install -Dhadoop.profile=2.0
for this to work. -->
<hbase.version>0.94.1</hbase.version>
+ <commons-lang.version>2.5</commons-lang.version>
+ <protobuf-java.version>2.4.0a</protobuf-java.version>
+ <slf4j.version>1.6.1</slf4j.version>
</properties>
<dependencyManagement>
<dependencies>