You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/05/29 05:11:49 UTC
git commit: CRUNCH-403: Use ___ instead of _ as substitute for $ in
Avro reflected classes in Scrunch
Repository: crunch
Updated Branches:
refs/heads/master 077343a5d -> a62a24b6f
CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/a62a24b6
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/a62a24b6
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/a62a24b6
Branch: refs/heads/master
Commit: a62a24b6f80666d92cac2b6dfa5c088e51f26ef6
Parents: 077343a
Author: Josh Wills <jw...@apache.org>
Authored: Tue May 27 12:34:11 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Wed May 28 20:07:36 2014 -0700
----------------------------------------------------------------------
.../org/apache/crunch/scrunch/PageRankClassTest.scala | 12 +++++-------
.../org/apache/crunch/scrunch/ScalaSafeReflectData.java | 4 ++--
2 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/a62a24b6/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
index 4cd9e84..3d3cb9f 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
@@ -27,14 +27,14 @@ import scala.collection.mutable.HashMap
import _root_.org.junit.Assert._
import _root_.org.junit.Test
-case class PageRankData(pr: Float, oldpr: Float, urls: Array[String]) {
+case class PageRankData(page_rank: Float, oldpr: Float, urls: Array[String]) {
def this() = this(0f, 0f, null)
- def scaledPageRank = pr / urls.length
+ def scaledPageRank = page_rank / urls.length
- def next(newPageRank: Float) = new PageRankData(newPageRank, pr, urls)
+ def next(newPageRank: Float) = new PageRankData(newPageRank, page_rank, urls)
- def delta = math.abs(pr - oldpr)
+ def delta = math.abs(page_rank - oldpr)
}
class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Float]] {
@@ -45,7 +45,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo
override def process(input: P[String, PageRankData], emitFn: Emitter[P[String, Float]]) {
val prd = input.second()
if (prd.urls.length > 0) {
- val newpr = prd.pr / prd.urls.length
+ val newpr = prd.page_rank / prd.urls.length
prd.urls.foreach(url => cache.put(url, cache(url) + newpr))
if (cache.size > 5000) {
cleanup(emitFn)
@@ -92,7 +92,6 @@ class PageRankClassTest extends CrunchSuite {
}
@Test def testPageRank {
- pipeline.getConfiguration.set("crunch.debug", "true")
var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
var delta = 1.0f
while (delta > 0.01f) {
@@ -104,7 +103,6 @@ class PageRankClassTest extends CrunchSuite {
}
def testFastPageRank {
- pipeline.getConfiguration.set("crunch.debug", "true")
var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
var delta = 1.0f
while (delta > 0.01f) {
http://git-wip-us.apache.org/repos/asf/crunch/blob/a62a24b6/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
index 7fbf52f..6118834 100644
--- a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
+++ b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java
@@ -236,11 +236,11 @@ public class ScalaSafeReflectData extends ReflectData.AllowNull {
}
private static String clean(String dirty) {
- return dirty.replace('$', '_');
+ return dirty.replace("$", "___");
}
private static String dirty(String clean) {
- return clean.replace('_', '$');
+ return clean.replace("___", "$");
}
// Return of this class and its superclasses to serialize.