You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2016/04/10 07:36:12 UTC
mahout git commit: NoJira: Remove comment references to Spark and fix
the Javadocs
Repository: mahout
Updated Branches:
refs/heads/flink-binding 430310dbf -> 681d30e54
NoJira: Remove comment references to Spark and fix the Javadocs
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/681d30e5
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/681d30e5
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/681d30e5
Branch: refs/heads/flink-binding
Commit: 681d30e54f1f90a75733990056f864ab6c7c9cc6
Parents: 430310d
Author: smarthi <sm...@apache.org>
Authored: Sun Apr 10 01:34:34 2016 -0400
Committer: smarthi <sm...@apache.org>
Committed: Sun Apr 10 01:34:34 2016 -0400
----------------------------------------------------------------------
.../mahout/flinkbindings/blas/FlinkOpAewB.scala | 3 +--
.../flinkbindings/blas/FlinkOpAewScalar.scala | 4 +---
.../mahout/flinkbindings/blas/FlinkOpAt.scala | 3 +--
.../mahout/flinkbindings/blas/FlinkOpAtA.scala | 5 ++---
.../mahout/flinkbindings/blas/FlinkOpAtB.scala | 3 +--
.../mahout/flinkbindings/blas/FlinkOpAx.scala | 3 +--
.../mahout/flinkbindings/blas/FlinkOpCBind.scala | 19 +++----------------
.../flinkbindings/blas/FlinkOpMapBlock.scala | 3 +--
.../mahout/flinkbindings/blas/FlinkOpRBind.scala | 5 +++--
.../flinkbindings/blas/FlinkOpRowRange.scala | 3 +--
.../blas/FlinkOpTimesRightMatrix.scala | 4 +---
.../mahout/flinkbindings/io/HDFSPathSearch.scala | 3 +--
.../mahout/flinkbindings/io/HDFSUtil.scala | 1 -
.../flinkbindings/io/Hadoop2HDFSUtil.scala | 11 +++--------
.../apache/mahout/flinkbindings/package.scala | 2 +-
15 files changed, 21 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewB.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewB.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewB.scala
index c61074b..a3be618 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewB.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewB.scala
@@ -10,8 +10,7 @@ import org.apache.mahout.math.drm.logical.OpAewB
import org.apache.mahout.math.scalabindings.RLikeOps._
/**
- * Implementation is inspired by Spark-binding's OpAewB
- * (see https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/AewB.scala)
+ * Implementation of Flink OpAewB
*/
object FlinkOpAewB {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewScalar.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewScalar.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewScalar.scala
index 56e7deb..6b034b8 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewScalar.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAewScalar.scala
@@ -18,7 +18,6 @@
*/
package org.apache.mahout.flinkbindings.blas
-import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.mahout.flinkbindings.drm.{BlockifiedFlinkDrm, FlinkDrm}
import org.apache.mahout.math.Matrix
@@ -31,8 +30,7 @@ import scala.reflect.ClassTag
import org.apache.flink.api.scala._
/**
- * Implementation is inspired by Spark-binding's OpAewScalar
- * (see https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/AewB.scala)
+ * Implementation if Flink OpAewScalar
*/
object FlinkOpAewScalar {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAt.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAt.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAt.scala
index 6e320af..45214e5 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAt.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAt.scala
@@ -27,8 +27,7 @@ import org.apache.mahout.math.scalabindings.RLikeOps._
import scala.Array.canBuildFrom
/**
- * Implementation is taken from Spark's At
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/At.scala
+ * Implementation of Flink At
*/
object FlinkOpAt {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala
index 6d0221a..4dc265f 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala
@@ -38,9 +38,8 @@ import scala.collection.JavaConverters._
import scala.collection._
/**
- * Inspired by Spark's implementation from
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/AtA.scala
- *
+ * Implementation of Flink A' * A
+ *
*/
object FlinkOpAtA {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtB.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtB.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtB.scala
index 0a2683c..c10c2e4 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtB.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtB.scala
@@ -34,8 +34,7 @@ import org.apache.mahout.math.scalabindings.RLikeOps._
import scala.collection.JavaConverters.asScalaBufferConverter
/**
- * Implementation is taken from Spark's AtB
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/AtB.scala
+ * Implementation of Flink A' * B
*/
object FlinkOpAtB {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAx.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAx.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAx.scala
index ca43b31..18a3c4b 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAx.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAx.scala
@@ -33,8 +33,7 @@ import org.apache.mahout.math.scalabindings._
import org.apache.mahout.math.{Matrix, Vector}
/**
- * Implementation is taken from Spark's Ax
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/Ax.scala
+ * Implementation of Flink Ax
*/
object FlinkOpAx {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpCBind.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpCBind.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpCBind.scala
index 65b2a25..17927a4 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpCBind.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpCBind.scala
@@ -18,30 +18,17 @@
*/
package org.apache.mahout.flinkbindings.blas
-import java.lang.Iterable
import org.apache.flink.api.common.typeinfo.TypeInformation
-
-import scala.collection.JavaConverters._
-import scala.collection.JavaConversions._
-import scala.reflect.ClassTag
-import org.apache.flink.api.common.functions.CoGroupFunction
-import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.scala._
-import org.apache.flink.util.Collector
-import org.apache.mahout.flinkbindings._
import org.apache.mahout.flinkbindings.drm._
import org.apache.mahout.math._
-import org.apache.mahout.math.drm.logical.OpCbind
-import org.apache.mahout.math.drm.logical.OpCbindScalar
+import org.apache.mahout.math.drm.logical.{OpCbind, OpCbindScalar}
import org.apache.mahout.math.scalabindings.RLikeOps._
-import com.google.common.collect.Lists
-import org.apache.mahout.flinkbindings.DrmDataSet
-import org.apache.mahout.math.scalabindings._
+import scala.collection.JavaConversions._
/**
- * Implementation is taken from Spark's cbind
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/CbindAB.scala
+ * Implementation of Flink's cbind
*/
object FlinkOpCBind {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpMapBlock.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpMapBlock.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpMapBlock.scala
index ec4769a..c22fa9a 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpMapBlock.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpMapBlock.scala
@@ -25,8 +25,7 @@ import org.apache.mahout.math.drm.logical.OpMapBlock
import org.apache.mahout.math.scalabindings.RLikeOps._
/**
- * Implementation is taken from Spark's MapBlock
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/MapBlock.scala
+ * Implementation of Flink's MapBlock
*/
object FlinkOpMapBlock {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRBind.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRBind.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRBind.scala
index 4fa2eaa..685486c 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRBind.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRBind.scala
@@ -20,14 +20,15 @@ package org.apache.mahout.flinkbindings.blas
import org.apache.flink.api.common.typeinfo.TypeInformation
-import scala.reflect.ClassTag
-
import org.apache.flink.api.scala.DataSet
import org.apache.mahout.flinkbindings.drm.FlinkDrm
import org.apache.mahout.flinkbindings.drm.RowsFlinkDrm
import org.apache.mahout.math.Vector
import org.apache.mahout.math.drm.logical.OpRbind
+/**
+ * Implementation of RBind
+ */
object FlinkOpRBind {
def rbind[K: TypeInformation](op: OpRbind[K], A: FlinkDrm[K], B: FlinkDrm[K]): FlinkDrm[K] = {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRowRange.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRowRange.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRowRange.scala
index 39f4ceb..c002002 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRowRange.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpRowRange.scala
@@ -24,8 +24,7 @@ import org.apache.mahout.flinkbindings.drm.RowsFlinkDrm
import org.apache.mahout.math.drm.logical.OpRowRange
/**
- * Implementation is taken from Spark's OpRowRange
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/Slicing.scala
+ * Implementation of Flink OpRowRange
*/
object FlinkOpRowRange {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala
index 70ad9d3..2ad7bd6 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala
@@ -30,8 +30,7 @@ import org.apache.mahout.math.scalabindings._
import org.apache.mahout.math.{DenseMatrix, Matrix}
/**
- * Implementation is taken from Spark's OpTimesRightMatrix:
- * https://github.com/apache/mahout/blob/master/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/AinCoreB.scala
+ * Implementation of OpTimesRightMatrix:
*/
object FlinkOpTimesRightMatrix {
@@ -55,7 +54,6 @@ object FlinkOpTimesRightMatrix {
override def open(params: Configuration): Unit = {
val runtime = this.getRuntimeContext
- //val dsB: java.util.List[Matrix]
val dsB: java.util.List[(Int, org.apache.mahout.math.Vector)] = runtime.getBroadcastVariable("matrix")
val m = dsB.size()
val n = dsB.get(0)._2.size
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSPathSearch.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSPathSearch.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSPathSearch.scala
index b9d9f1b..c0b6ee9 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSPathSearch.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSPathSearch.scala
@@ -24,12 +24,11 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
/**
* Returns a [[java.lang.String]], which is comma delimited list of URIs discovered based on parameters
* in the constructor.
- * The String is formatted to be input into [[org.apache.spark.SparkContext#textFile()]]
+ * The String is formatted to be input into [[org.apache.flink.api.scala.ExecutionEnvironment#textFile()]]
* @param pathURI Where to start looking for inFiles, may be a list of comma delimited URIs
* @param filePattern regex that must match the entire filename to have the file returned
* @param recursive true traverses the filesystem recursively, default = false
*
- * Copied from /spark/src/main/scala/org/apache/mahout/common
*/
case class HDFSPathSearch(pathURI: String, filePattern: String = "", recursive: Boolean = false) {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSUtil.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSUtil.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSUtil.scala
index d0d853d..73436f1 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSUtil.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/HDFSUtil.scala
@@ -21,7 +21,6 @@ package org.apache.mahout.flinkbindings.io
/**
* High level Hadoop version-specific hdfs manipulations we need in context of our operations.
*
- * Copied from /spark/src/main/scala/org/apache/mahout/common
*/
trait HDFSUtil {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/io/Hadoop2HDFSUtil.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/Hadoop2HDFSUtil.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/Hadoop2HDFSUtil.scala
index 50d3bc6..9b67913 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/io/Hadoop2HDFSUtil.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/io/Hadoop2HDFSUtil.scala
@@ -18,17 +18,12 @@
*/
package org.apache.mahout.flinkbindings.io
-import org.apache.hadoop.io.{ Writable, SequenceFile }
-import org.apache.hadoop.fs.{ FileSystem, Path }
import org.apache.hadoop.conf.Configuration
-import collection._
-import JavaConversions._
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.{SequenceFile, Writable}
/**
- * Deprecated Hadoop 1 api which we currently explicitly import via Mahout dependencies. May not work
- * with Hadoop 2.0
- *
- * Copied from /spark/src/main/scala/org/apache/mahout/common
+ * Deprecated Hadoop 1 api which we currently explicitly import via Mahout dependencies.
*/
object Hadoop2HDFSUtil extends HDFSUtil {
http://git-wip-us.apache.org/repos/asf/mahout/blob/681d30e5/flink/src/main/scala/org/apache/mahout/flinkbindings/package.scala
----------------------------------------------------------------------
diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/package.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/package.scala
index e769952..cf4da41 100644
--- a/flink/src/main/scala/org/apache/mahout/flinkbindings/package.scala
+++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/package.scala
@@ -65,7 +65,7 @@ package object flinkbindings {
new RowsFlinkDrm[K](flinkDrm.ds, flinkDrm.ncol)
}
- /** Adding Spark-specific ops */
+ /** Adding Flink-specific ops */
implicit def cpDrm2cpDrmFlinkOps[K: ClassTag](drm: CheckpointedDrm[K]): CheckpointedFlinkDrmOps[K] =
new CheckpointedFlinkDrmOps[K](drm)