You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2016/04/11 10:09:51 UTC
[24/50] [abbrv] mahout git commit: MAHOUT-1823: Modify
MahoutFlinkTestSuite to implement FlinkTestBase, this closes apache/mahout#210
MAHOUT-1823: Modify MahoutFlinkTestSuite to implement FlinkTestBase, this closes apache/mahout#210
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/c1d79ff4
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/c1d79ff4
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/c1d79ff4
Branch: refs/heads/master
Commit: c1d79ff4c2e38325736f09a8a35eb7ba2625509d
Parents: c75f3e2
Author: smarthi <sm...@apache.org>
Authored: Wed Apr 6 02:35:29 2016 -0400
Committer: smarthi <sm...@apache.org>
Committed: Wed Apr 6 02:35:29 2016 -0400
----------------------------------------------------------------------
flink/pom.xml | 11 +++++-
.../flinkbindings/DistributedFlinkSuite.scala | 39 +++++++++++++++++---
.../DistributedDecompositionsSuite.scala | 28 +++++++++-----
.../standard/DrmLikeOpsSuite.scala | 26 +++++++++----
.../flinkbindings/standard/DrmLikeSuite.scala | 26 +++++++++----
.../standard/RLikeDrmOpsSuite.scala | 19 +++++++++-
6 files changed, 117 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/pom.xml
----------------------------------------------------------------------
diff --git a/flink/pom.xml b/flink/pom.xml
index a75c7e2..e9b94cf 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -132,7 +132,16 @@
<artifactId>flink-clients_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_2.10</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-tests_2.10</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.mahout</groupId>
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala
----------------------------------------------------------------------
diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala
index 41c7a6a..480a256 100644
--- a/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala
+++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala
@@ -18,10 +18,15 @@
*/
package org.apache.mahout.flinkbindings
+import java.util.concurrent.TimeUnit
+
import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.test.util.{ForkableFlinkMiniCluster, TestBaseUtils}
import org.apache.mahout.math.drm.DistributedContext
import org.apache.mahout.test.DistributedMahoutSuite
-import org.scalatest.Suite
+import org.scalatest.{ConfigMap, Suite}
+
+import scala.concurrent.duration.FiniteDuration
trait DistributedFlinkSuite extends DistributedMahoutSuite { this: Suite =>
@@ -29,10 +34,12 @@ trait DistributedFlinkSuite extends DistributedMahoutSuite { this: Suite =>
protected implicit var mahoutCtx: DistributedContext = _
protected var env: ExecutionEnvironment = null
+ var cluster: Option[ForkableFlinkMiniCluster] = None
+ val parallelism = 4
+ protected val DEFAULT_AKKA_ASK_TIMEOUT: Long = 1000
+ protected var DEFAULT_TIMEOUT: FiniteDuration = new FiniteDuration(DEFAULT_AKKA_ASK_TIMEOUT, TimeUnit.SECONDS)
+
def initContext() {
- env = ExecutionEnvironment.getExecutionEnvironment
- // set this higher so that tests like dsqDist(X,Y) have enough available slots to pass on a single machine.
- env.setParallelism(10)
mahoutCtx = wrapContext(env)
}
@@ -45,4 +52,26 @@ trait DistributedFlinkSuite extends DistributedMahoutSuite { this: Suite =>
// env.execute("Mahout Flink Binding Test Suite")
}
-}
\ No newline at end of file
+ override protected def afterAll(configMap: ConfigMap): Unit = {
+ super.afterAll(configMap)
+// resetContext()
+ cluster.foreach(c => TestBaseUtils.stopCluster(c, DEFAULT_TIMEOUT))
+ }
+
+ override protected def beforeAll(configMap: ConfigMap): Unit = {
+ super.beforeAll(configMap)
+
+ val cl = TestBaseUtils.startCluster(
+ 1,
+ parallelism,
+ false,
+ false,
+ true)
+
+ env = ExecutionEnvironment.createLocalEnvironment(parallelism)
+
+ cluster = Some(cl)
+ initContext()
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DistributedDecompositionsSuite.scala
----------------------------------------------------------------------
diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DistributedDecompositionsSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DistributedDecompositionsSuite.scala
index 82ca3ff..4256b52 100644
--- a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DistributedDecompositionsSuite.scala
+++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DistributedDecompositionsSuite.scala
@@ -1,16 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.mahout.flinkbindings.standard
import org.apache.mahout.flinkbindings._
-import org.apache.mahout.math._
-import org.apache.mahout.math.drm._
-import org.apache.mahout.math.drm.RLikeDrmOps._
-import org.apache.mahout.math.scalabindings._
-import org.apache.mahout.math.scalabindings.RLikeOps._
-import org.junit.runner.RunWith
-import org.scalatest.FunSuite
-import org.scalatest.junit.JUnitRunner
import org.apache.mahout.math.decompositions.DistributedDecompositionsSuiteBase
-
+import org.scalatest.FunSuite
class DistributedDecompositionsSuite extends FunSuite with DistributedFlinkSuite
with DistributedDecompositionsSuiteBase {
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeOpsSuite.scala
----------------------------------------------------------------------
diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeOpsSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeOpsSuite.scala
index 325d118..3752187 100644
--- a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeOpsSuite.scala
+++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeOpsSuite.scala
@@ -1,16 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.mahout.flinkbindings.standard
import org.apache.mahout.flinkbindings._
-import org.apache.mahout.math._
import org.apache.mahout.math.drm._
-import org.apache.mahout.math.drm.RLikeDrmOps._
-import org.apache.mahout.math.scalabindings._
-import org.apache.mahout.math.scalabindings.RLikeOps._
-import org.junit.runner.RunWith
import org.scalatest.FunSuite
-import org.scalatest.junit.JUnitRunner
-import org.apache.mahout.math.decompositions.DistributedDecompositionsSuiteBase
-
class DrmLikeOpsSuite extends FunSuite with DistributedFlinkSuite
with DrmLikeOpsSuiteBase {
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeSuite.scala
----------------------------------------------------------------------
diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeSuite.scala
index dfa7360..0a1653b 100644
--- a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeSuite.scala
+++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/DrmLikeSuite.scala
@@ -1,16 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.mahout.flinkbindings.standard
import org.apache.mahout.flinkbindings._
-import org.apache.mahout.math._
import org.apache.mahout.math.drm._
-import org.apache.mahout.math.drm.RLikeDrmOps._
-import org.apache.mahout.math.scalabindings._
-import org.apache.mahout.math.scalabindings.RLikeOps._
-import org.junit.runner.RunWith
import org.scalatest.FunSuite
-import org.scalatest.junit.JUnitRunner
-import org.apache.mahout.math.decompositions.DistributedDecompositionsSuiteBase
-
class DrmLikeSuite extends FunSuite with DistributedFlinkSuite
with DrmLikeSuiteBase {
http://git-wip-us.apache.org/repos/asf/mahout/blob/c1d79ff4/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/RLikeDrmOpsSuite.scala
----------------------------------------------------------------------
diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/RLikeDrmOpsSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/RLikeDrmOpsSuite.scala
index c0ff76c..8bb1b02 100644
--- a/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/RLikeDrmOpsSuite.scala
+++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/standard/RLikeDrmOpsSuite.scala
@@ -1,10 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.mahout.flinkbindings.standard
import org.apache.mahout.flinkbindings._
import org.apache.mahout.math.drm._
import org.scalatest.FunSuite
-
class RLikeDrmOpsSuite extends FunSuite with DistributedFlinkSuite
with RLikeDrmOpsSuiteBase {