You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2015/09/03 09:28:59 UTC
[23/25] gora git commit: GORA-386 Gora Spark Backend Support addition
to make forbidden api compliant
GORA-386 Gora Spark Backend Support addition to make forbidden api compliant
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/2ce2fda5
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/2ce2fda5
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/2ce2fda5
Branch: refs/heads/master
Commit: 2ce2fda5b7f688f1c5fa9e10f7c8ff64ce99be12
Parents: 965c449
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Tue Sep 1 15:26:59 2015 -0700
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Tue Sep 1 15:26:59 2015 -0700
----------------------------------------------------------------------
.../apache/gora/examples/spark/SparkWordCount.java | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/2ce2fda5/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java
----------------------------------------------------------------------
diff --git a/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java b/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java
index 0a84d58..4f91fb2 100644
--- a/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java
+++ b/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java
@@ -17,6 +17,8 @@
*/
package org.apache.gora.examples.spark;
+import java.nio.charset.Charset;
+
import org.apache.gora.examples.generated.TokenDatum;
import org.apache.gora.examples.generated.WebPage;
import org.apache.gora.spark.GoraSparkEngine;
@@ -52,7 +54,7 @@ public class SparkWordCount {
@Override
public Tuple2<String, Long> call(WebPage webPage)
throws Exception {
- String content = new String(webPage.getContent().array());
+ String content = new String(webPage.getContent().array(), Charset.defaultCharset());
return new Tuple2<>(content, 1L);
}
};
@@ -86,20 +88,20 @@ public class SparkWordCount {
JavaPairRDD<String, WebPage> goraRDD = goraSparkEngine.initialize(sc, inStore);
long count = goraRDD.count();
- System.out.println("Total Web page count: " + count);
+ log.info("Total Web page count: {}", count);
JavaRDD<Tuple2<String, Long>> mappedGoraRdd = goraRDD.values().map(mapFunc);
JavaPairRDD<String, Long> reducedGoraRdd = JavaPairRDD.fromJavaRDD(mappedGoraRdd).reduceByKey(redFunc);
//Print output for debug purpose
- System.out.println("SparkWordCount debug purpose TokenDatum print starts:");
+ log.info("SparkWordCount debug purpose TokenDatum print starts:");
Map<String, Long> tokenDatumMap = reducedGoraRdd.collectAsMap();
for (String key : tokenDatumMap.keySet()) {
- System.out.println(key);
- System.out.println(tokenDatumMap.get(key));
+ log.info(key);
+ log.info(tokenDatumMap.get(key).toString());
}
- System.out.println("SparkWordCount debug purpose TokenDatum print ends:");
+ log.info("SparkWordCount debug purpose TokenDatum print ends:");
//
//write output to datastore
@@ -134,7 +136,7 @@ public class SparkWordCount {
public static void main(String[] args) throws Exception {
if (args.length < 2) {
- System.err.println(USAGE);
+ log.info(USAGE);
System.exit(1);
}