You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/06/03 03:22:31 UTC
[1/4] git commit: CRUNCH-408: Fix the HFileSource globStatus check to
work on Hadoop2
Repository: crunch
Updated Branches:
refs/heads/apache-crunch-0.10 [created] 4c364d0df
CRUNCH-408: Fix the HFileSource globStatus check to work on Hadoop2
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/2e284058
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/2e284058
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/2e284058
Branch: refs/heads/apache-crunch-0.10
Commit: 2e284058a915482c922e6e4328f08674ac4724f1
Parents: 35d404c
Author: Josh Wills <jw...@apache.org>
Authored: Sun Jun 1 20:41:01 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Sun Jun 1 20:42:32 2014 -0700
----------------------------------------------------------------------
.../org/apache/crunch/io/hbase/HFileSource.java | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/2e284058/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
index c21cc47..47abe9a 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
@@ -120,10 +120,28 @@ public class HFileSource extends FileSourceImpl<KeyValue> implements ReadableSou
long sum = 0;
for (Path path : getPaths()) {
try {
- sum += SourceTargetHelper.getPathSize(conf, new Path(path, "*"));
+ sum += getSizeInternal(conf, path);
} catch (IOException e) {
LOG.warn("Failed to estimate size of " + path);
}
+ System.out.println("Size after read of path = " + path.toString() + " = " + sum);
+ }
+ return sum;
+ }
+
+ private long getSizeInternal(Configuration conf, Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus[] statuses = fs.globStatus(path, HFileInputFormat.HIDDEN_FILE_FILTER);
+ if (statuses == null) {
+ return 0;
+ }
+ long sum = 0;
+ for (FileStatus status : statuses) {
+ if (status.isDir()) {
+ sum += SourceTargetHelper.getPathSize(fs, status.getPath());
+ } else {
+ sum += status.getLen();
+ }
}
return sum;
}
[4/4] git commit: [maven-release-plugin] prepare for next development
iteration
Posted by jw...@apache.org.
[maven-release-plugin] prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/4c364d0d
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/4c364d0d
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/4c364d0d
Branch: refs/heads/apache-crunch-0.10
Commit: 4c364d0dff2e04a3a8609240ac96869386d0789c
Parents: 22714f5
Author: Josh Wills <jw...@apache.org>
Authored: Mon Jun 2 18:08:00 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Mon Jun 2 18:08:00 2014 -0700
----------------------------------------------------------------------
crunch-archetype/pom.xml | 2 +-
crunch-contrib/pom.xml | 2 +-
crunch-core/pom.xml | 2 +-
crunch-dist/pom.xml | 2 +-
crunch-examples/pom.xml | 2 +-
crunch-hbase/pom.xml | 2 +-
crunch-scrunch/pom.xml | 2 +-
crunch-spark/pom.xml | 2 +-
crunch-test/pom.xml | 2 +-
pom.xml | 4 ++--
10 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-archetype/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-archetype/pom.xml b/crunch-archetype/pom.xml
index 6f4bc00..691eae3 100644
--- a/crunch-archetype/pom.xml
+++ b/crunch-archetype/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-archetype</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-contrib/pom.xml b/crunch-contrib/pom.xml
index 053bedb..be6f5d8 100644
--- a/crunch-contrib/pom.xml
+++ b/crunch-contrib/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-contrib</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index ae3f3e6..9c0013a 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-core</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-dist/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-dist/pom.xml b/crunch-dist/pom.xml
index a3ce391..96602f7 100644
--- a/crunch-dist/pom.xml
+++ b/crunch-dist/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-examples/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-examples/pom.xml b/crunch-examples/pom.xml
index d3d4899..21fe017 100644
--- a/crunch-examples/pom.xml
+++ b/crunch-examples/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-hbase/pom.xml b/crunch-hbase/pom.xml
index 69ae44d..5bc7ba0 100644
--- a/crunch-hbase/pom.xml
+++ b/crunch-hbase/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-hbase</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml
index d0b839a..88318af 100644
--- a/crunch-scrunch/pom.xml
+++ b/crunch-scrunch/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-scrunch</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-spark/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-spark/pom.xml b/crunch-spark/pom.xml
index 1487341..250b62b 100644
--- a/crunch-spark/pom.xml
+++ b/crunch-spark/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-spark</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/crunch-test/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index 97f497a..dca00fc 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
</parent>
<artifactId>crunch-test</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/4c364d0d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c2bce21..f0e6a27 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,7 +27,7 @@ under the License.
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0</version>
+ <version>0.10.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Crunch</name>
@@ -101,7 +101,7 @@ under the License.
<url>https://git-wip-us.apache.org/repos/asf?p=crunch.git</url>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/crunch.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/crunch.git</developerConnection>
- <tag>apache-crunch-0.10.0</tag>
+ <tag>apache-crunch-0.10</tag>
</scm>
<issueManagement>
[3/4] git commit: [maven-release-plugin] prepare release
apache-crunch-0.10.0
Posted by jw...@apache.org.
[maven-release-plugin] prepare release apache-crunch-0.10.0
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/22714f50
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/22714f50
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/22714f50
Branch: refs/heads/apache-crunch-0.10
Commit: 22714f5010734175296e6fc8628ec505bd4433cd
Parents: 0aad015
Author: Josh Wills <jw...@apache.org>
Authored: Mon Jun 2 18:07:59 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Mon Jun 2 18:07:59 2014 -0700
----------------------------------------------------------------------
crunch-archetype/pom.xml | 2 +-
crunch-contrib/pom.xml | 2 +-
crunch-core/pom.xml | 2 +-
crunch-dist/pom.xml | 2 +-
crunch-examples/pom.xml | 2 +-
crunch-hbase/pom.xml | 2 +-
crunch-scrunch/pom.xml | 2 +-
crunch-spark/pom.xml | 2 +-
crunch-test/pom.xml | 2 +-
pom.xml | 4 ++--
10 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-archetype/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-archetype/pom.xml b/crunch-archetype/pom.xml
index 41d38fc..6f4bc00 100644
--- a/crunch-archetype/pom.xml
+++ b/crunch-archetype/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-archetype</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-contrib/pom.xml b/crunch-contrib/pom.xml
index 3cdc890..053bedb 100644
--- a/crunch-contrib/pom.xml
+++ b/crunch-contrib/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-contrib</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-core/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index 06a8a19..ae3f3e6 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-core</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-dist/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-dist/pom.xml b/crunch-dist/pom.xml
index 090e852..a3ce391 100644
--- a/crunch-dist/pom.xml
+++ b/crunch-dist/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-examples/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-examples/pom.xml b/crunch-examples/pom.xml
index 04f9cdf..d3d4899 100644
--- a/crunch-examples/pom.xml
+++ b/crunch-examples/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-hbase/pom.xml b/crunch-hbase/pom.xml
index b97e063..69ae44d 100644
--- a/crunch-hbase/pom.xml
+++ b/crunch-hbase/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-hbase</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml
index 14b69d1..d0b839a 100644
--- a/crunch-scrunch/pom.xml
+++ b/crunch-scrunch/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-scrunch</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-spark/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-spark/pom.xml b/crunch-spark/pom.xml
index 9a1d485..1487341 100644
--- a/crunch-spark/pom.xml
+++ b/crunch-spark/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-spark</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/crunch-test/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index 5df6ae1..97f497a 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -22,7 +22,7 @@ under the License.
<parent>
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
</parent>
<artifactId>crunch-test</artifactId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/22714f50/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1d94f77..c2bce21 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,7 +27,7 @@ under the License.
<groupId>org.apache.crunch</groupId>
<artifactId>crunch-parent</artifactId>
- <version>0.10.0-SNAPSHOT</version>
+ <version>0.10.0</version>
<packaging>pom</packaging>
<name>Apache Crunch</name>
@@ -101,7 +101,7 @@ under the License.
<url>https://git-wip-us.apache.org/repos/asf?p=crunch.git</url>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/crunch.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/crunch.git</developerConnection>
- <tag>apache-crunch-0.10</tag>
+ <tag>apache-crunch-0.10.0</tag>
</scm>
<issueManagement>
[2/4] git commit: CRUNCH-411: Drop _2.10 extension from Scrunch and
crunch-spark targets
Posted by jw...@apache.org.
CRUNCH-411: Drop _2.10 extension from Scrunch and crunch-spark targets
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/0aad0153
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/0aad0153
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/0aad0153
Branch: refs/heads/apache-crunch-0.10
Commit: 0aad01536ac5e7a0a38dbfc4f920217e152175a1
Parents: 2e28405
Author: Josh Wills <jw...@apache.org>
Authored: Mon Jun 2 17:46:24 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Mon Jun 2 17:51:19 2014 -0700
----------------------------------------------------------------------
crunch-dist/pom.xml | 2 +-
crunch-scrunch/pom.xml | 2 +-
crunch-spark/pom.xml | 2 +-
pom.xml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/0aad0153/crunch-dist/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-dist/pom.xml b/crunch-dist/pom.xml
index 437bf93..090e852 100644
--- a/crunch-dist/pom.xml
+++ b/crunch-dist/pom.xml
@@ -55,7 +55,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.crunch</groupId>
- <artifactId>crunch-scrunch_${scala.base.version}</artifactId>
+ <artifactId>crunch-scrunch</artifactId>
</dependency>
<dependency>
<groupId>org.apache.crunch</groupId>
http://git-wip-us.apache.org/repos/asf/crunch/blob/0aad0153/crunch-scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml
index 57c1b51..14b69d1 100644
--- a/crunch-scrunch/pom.xml
+++ b/crunch-scrunch/pom.xml
@@ -25,7 +25,7 @@ under the License.
<version>0.10.0-SNAPSHOT</version>
</parent>
- <artifactId>crunch-scrunch_${scala.base.version}</artifactId>
+ <artifactId>crunch-scrunch</artifactId>
<name>Apache Crunch for Scala</name>
<dependencies>
http://git-wip-us.apache.org/repos/asf/crunch/blob/0aad0153/crunch-spark/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-spark/pom.xml b/crunch-spark/pom.xml
index 0712f2d..9a1d485 100644
--- a/crunch-spark/pom.xml
+++ b/crunch-spark/pom.xml
@@ -25,7 +25,7 @@ under the License.
<version>0.10.0-SNAPSHOT</version>
</parent>
- <artifactId>crunch-spark_${scala.base.version}</artifactId>
+ <artifactId>crunch-spark</artifactId>
<name>Apache Crunch for Spark</name>
<dependencies>
http://git-wip-us.apache.org/repos/asf/crunch/blob/0aad0153/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index da9bcc8..1d94f77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,7 +143,7 @@ under the License.
<dependency>
<groupId>org.apache.crunch</groupId>
- <artifactId>crunch-scrunch_${scala.base.version}</artifactId>
+ <artifactId>crunch-scrunch</artifactId>
<version>${project.version}</version>
</dependency>