You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/14 07:59:50 UTC
[26/50] git commit: Made more things private.
Made more things private.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/87f335db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/87f335db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/87f335db
Branch: refs/heads/master
Commit: 87f335db78221fc250bd64f39a334293db490379
Parents: a4e12af
Author: Reynold Xin <rx...@apache.org>
Authored: Mon Jan 13 18:30:26 2014 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Mon Jan 13 18:30:26 2014 -0800
----------------------------------------------------------------------
graphx/src/main/scala/org/apache/spark/graphx/Edge.scala | 8 ++++----
.../main/scala/org/apache/spark/graphx/GraphOps.scala | 4 ----
.../org/apache/spark/graphx/impl/EdgePartition.scala | 1 +
.../apache/spark/graphx/impl/EdgePartitionBuilder.scala | 1 +
.../scala/org/apache/spark/graphx/impl/GraphImpl.scala | 2 +-
.../apache/spark/graphx/impl/MessageToPartition.scala | 5 +++++
.../apache/spark/graphx/impl/ReplicatedVertexView.scala | 3 ++-
.../org/apache/spark/graphx/impl/RoutingTable.scala | 1 +
.../scala/org/apache/spark/graphx/impl/Serializers.scala | 11 ++++++++++-
.../scala/org/apache/spark/graphx/impl/package.scala | 2 +-
10 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/Edge.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Edge.scala b/graphx/src/main/scala/org/apache/spark/graphx/Edge.scala
index 6c396c3..8546305 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/Edge.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/Edge.scala
@@ -1,18 +1,18 @@
package org.apache.spark.graphx
-
/**
* A single directed edge consisting of a source id, target id,
* and the data associated with the edge.
*
* @tparam ED type of the edge attribute
+ *
+ * @param srcId The vertex id of the source vertex
+ * @param dstId The vertex id of the target vertex
+ * @param attr The attribute associated with the edge
*/
case class Edge[@specialized(Char, Int, Boolean, Byte, Long, Float, Double) ED] (
- /** The vertex id of the source vertex */
var srcId: VertexID = 0,
- /** The vertex id of the target vertex. */
var dstId: VertexID = 0,
- /** The attribute associated with the edge. */
var attr: ED = null.asInstanceOf[ED])
extends Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala b/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
index 66d5180..447ef55 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
@@ -209,12 +209,8 @@ class GraphOps[VD: ClassTag, ED: ClassTag](graph: Graph[VD, ED]) {
* This function iterates until there are no remaining messages, or
* for `maxIterations` iterations.
*
- * @tparam VD the vertex data type
- * @tparam ED the edge data type
* @tparam A the Pregel message type
*
- * @param graph the input graph.
- *
* @param initialMsg the message each vertex will receive at the on
* the first iteration
*
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartition.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartition.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartition.scala
index d4d7162..ee95ead 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartition.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartition.scala
@@ -15,6 +15,7 @@ import org.apache.spark.graphx.util.collection.PrimitiveKeyOpenHashMap
* @param index a clustered index on source vertex id
* @tparam ED the edge attribute type.
*/
+private[graphx]
class EdgePartition[@specialized(Char, Int, Boolean, Byte, Long, Float, Double) ED: ClassTag](
val srcIds: Array[VertexID],
val dstIds: Array[VertexID],
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartitionBuilder.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartitionBuilder.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartitionBuilder.scala
index fbc2940..9d072f9 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartitionBuilder.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgePartitionBuilder.scala
@@ -7,6 +7,7 @@ import org.apache.spark.graphx._
import org.apache.spark.graphx.util.collection.PrimitiveKeyOpenHashMap
import org.apache.spark.util.collection.PrimitiveVector
+private[graphx]
class EdgePartitionBuilder[@specialized(Long, Int, Double) ED: ClassTag](size: Int = 64) {
var edges = new PrimitiveVector[Edge[ED]](size)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
index 97ca642..348490c 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
@@ -15,7 +15,7 @@ import org.apache.spark.util.ClosureCleaner
/**
- * A Graph RDD that supports computation on graphs.
+ * A graph that supports computation on graphs.
*
* Graphs are represented using two classes of data: vertex-partitioned and
* edge-partitioned. `vertices` contains vertex attributes, which are vertex-partitioned. `edges`
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala
index ad5daf8..05508ff 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/MessageToPartition.scala
@@ -7,6 +7,7 @@ import org.apache.spark.graphx.{PartitionID, VertexID}
import org.apache.spark.rdd.{ShuffledRDD, RDD}
+private[graphx]
class VertexBroadcastMsg[@specialized(Int, Long, Double, Boolean) T](
@transient var partition: PartitionID,
var vid: VertexID,
@@ -26,6 +27,7 @@ class VertexBroadcastMsg[@specialized(Int, Long, Double, Boolean) T](
* @param partition index of the target partition.
* @param data value to send
*/
+private[graphx]
class MessageToPartition[@specialized(Int, Long, Double, Char, Boolean/*, AnyRef*/) T](
@transient var partition: PartitionID,
var data: T)
@@ -39,6 +41,7 @@ class MessageToPartition[@specialized(Int, Long, Double, Char, Boolean/*, AnyRef
}
+private[graphx]
class VertexBroadcastMsgRDDFunctions[T: ClassTag](self: RDD[VertexBroadcastMsg[T]]) {
def partitionBy(partitioner: Partitioner): RDD[VertexBroadcastMsg[T]] = {
val rdd = new ShuffledRDD[PartitionID, (VertexID, T), VertexBroadcastMsg[T]](self, partitioner)
@@ -56,6 +59,7 @@ class VertexBroadcastMsgRDDFunctions[T: ClassTag](self: RDD[VertexBroadcastMsg[T
}
+private[graphx]
class MsgRDDFunctions[T: ClassTag](self: RDD[MessageToPartition[T]]) {
/**
@@ -68,6 +72,7 @@ class MsgRDDFunctions[T: ClassTag](self: RDD[MessageToPartition[T]]) {
}
+private[graphx]
object MsgRDDFunctions {
implicit def rdd2PartitionRDDFunctions[T: ClassTag](rdd: RDD[MessageToPartition[T]]) = {
new MsgRDDFunctions(rdd)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/ReplicatedVertexView.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/ReplicatedVertexView.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/ReplicatedVertexView.scala
index 0e2f5a9..4ebe0b0 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/ReplicatedVertexView.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/ReplicatedVertexView.scala
@@ -138,7 +138,7 @@ class ReplicatedVertexView[VD: ClassTag](
}
}
-object ReplicatedVertexView {
+private object ReplicatedVertexView {
protected def buildBuffer[VD: ClassTag](
pid2vidIter: Iterator[Array[Array[VertexID]]],
vertexPartIter: Iterator[VertexPartition[VD]]) = {
@@ -187,6 +187,7 @@ object ReplicatedVertexView {
}
}
+private[graphx]
class VertexAttributeBlock[VD: ClassTag](val vids: Array[VertexID], val attrs: Array[VD])
extends Serializable {
def iterator: Iterator[(VertexID, VD)] =
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/RoutingTable.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/RoutingTable.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/RoutingTable.scala
index 3bd8b24..f342fd7 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/RoutingTable.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/RoutingTable.scala
@@ -12,6 +12,7 @@ import org.apache.spark.util.collection.PrimitiveVector
* may be used multiple times in ReplicatedVertexView -- once to ship the vertex attributes and
* (possibly) once to ship the active-set information.
*/
+private[impl]
class RoutingTable(edges: EdgeRDD[_], vertices: VertexRDD[_]) {
val bothAttrs: RDD[Array[Array[VertexID]]] = createPid2Vid(true, true)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala
index 1c3c87f..cbd6318 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/Serializers.scala
@@ -7,6 +7,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.graphx._
import org.apache.spark.serializer._
+private[graphx]
class VertexIDMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -27,6 +28,7 @@ class VertexIDMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for VertexBroadcastMessage[Int]. */
+private[graphx]
class IntVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -50,6 +52,7 @@ class IntVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for VertexBroadcastMessage[Long]. */
+private[graphx]
class LongVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -73,6 +76,7 @@ class LongVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for VertexBroadcastMessage[Double]. */
+private[graphx]
class DoubleVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -96,6 +100,7 @@ class DoubleVertexBroadcastMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for AggregationMessage[Int]. */
+private[graphx]
class IntAggMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -119,6 +124,7 @@ class IntAggMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for AggregationMessage[Long]. */
+private[graphx]
class LongAggMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -142,6 +148,7 @@ class LongAggMsgSerializer(conf: SparkConf) extends Serializer {
}
/** A special shuffle serializer for AggregationMessage[Double]. */
+private[graphx]
class DoubleAggMsgSerializer(conf: SparkConf) extends Serializer {
override def newInstance(): SerializerInstance = new ShuffleSerializerInstance {
@@ -168,6 +175,7 @@ class DoubleAggMsgSerializer(conf: SparkConf) extends Serializer {
// Helper classes to shorten the implementation of those special serializers.
////////////////////////////////////////////////////////////////////////////////
+private[graphx]
abstract class ShuffleSerializationStream(s: OutputStream) extends SerializationStream {
// The implementation should override this one.
def writeObject[T](t: T): SerializationStream
@@ -281,6 +289,7 @@ abstract class ShuffleSerializationStream(s: OutputStream) extends Serialization
override def close(): Unit = s.close()
}
+private[graphx]
abstract class ShuffleDeserializationStream(s: InputStream) extends DeserializationStream {
// The implementation should override this one.
def readObject[T](): T
@@ -371,7 +380,7 @@ abstract class ShuffleDeserializationStream(s: InputStream) extends Deserializat
override def close(): Unit = s.close()
}
-sealed trait ShuffleSerializerInstance extends SerializerInstance {
+private[graphx] sealed trait ShuffleSerializerInstance extends SerializerInstance {
override def serialize[T](t: T): ByteBuffer = throw new UnsupportedOperationException
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/87f335db/graphx/src/main/scala/org/apache/spark/graphx/impl/package.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/package.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/package.scala
index a6bbf63..cfc3281 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/package.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/package.scala
@@ -3,5 +3,5 @@ package org.apache.spark.graphx
import org.apache.spark.util.collection.OpenHashSet
package object impl {
- type VertexIdToIndexMap = OpenHashSet[VertexID]
+ private[graphx] type VertexIdToIndexMap = OpenHashSet[VertexID]
}