You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2018/01/23 00:49:57 UTC
[1/3] incubator-hivemall git commit: Added missing
HivemallStreamingOps.scala for Spark 2.2 module
Repository: incubator-hivemall
Updated Branches:
refs/heads/v0.5.0 30a22e09e -> 26e3d705f
Added missing HivemallStreamingOps.scala for Spark 2.2 module
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/1b82b218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/1b82b218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/1b82b218
Branch: refs/heads/v0.5.0
Commit: 1b82b218b6c8733a80c518edb8583074750bb793
Parents: 30a22e0
Author: Makoto Yui <my...@apache.org>
Authored: Tue Jan 23 09:47:41 2018 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Tue Jan 23 09:47:41 2018 +0900
----------------------------------------------------------------------
.../spark/streaming/HivemallStreamingOps.scala | 47 ++++++++++++++++++++
1 file changed, 47 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/1b82b218/spark/spark-2.2/src/main/scala/org/apache/spark/streaming/HivemallStreamingOps.scala
----------------------------------------------------------------------
diff --git a/spark/spark-2.2/src/main/scala/org/apache/spark/streaming/HivemallStreamingOps.scala b/spark/spark-2.2/src/main/scala/org/apache/spark/streaming/HivemallStreamingOps.scala
new file mode 100644
index 0000000..a6bbb4b
--- /dev/null
+++ b/spark/spark-2.2/src/main/scala/org/apache/spark/streaming/HivemallStreamingOps.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.streaming
+
+import scala.reflect.ClassTag
+
+import org.apache.spark.ml.feature.HivemallLabeledPoint
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row, SQLContext}
+import org.apache.spark.streaming.dstream.DStream
+
+final class HivemallStreamingOps(ds: DStream[HivemallLabeledPoint]) {
+
+ def predict[U: ClassTag](f: DataFrame => DataFrame)(implicit sqlContext: SQLContext)
+ : DStream[Row] = {
+ ds.transform[Row] { rdd: RDD[HivemallLabeledPoint] =>
+ f(sqlContext.createDataFrame(rdd)).rdd
+ }
+ }
+}
+
+object HivemallStreamingOps {
+
+ /**
+ * Implicitly inject the [[HivemallStreamingOps]] into [[DStream]].
+ */
+ implicit def dataFrameToHivemallStreamingOps(ds: DStream[HivemallLabeledPoint])
+ : HivemallStreamingOps = {
+ new HivemallStreamingOps(ds)
+ }
+}
[2/3] incubator-hivemall git commit: Changed spark module artifact
names
Posted by my...@apache.org.
Changed spark module artifact names
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/afb2774e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/afb2774e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/afb2774e
Branch: refs/heads/v0.5.0
Commit: afb2774e2cd7f7f21dd7b0bc6303f595b7a1291b
Parents: 1b82b21
Author: Makoto Yui <my...@apache.org>
Authored: Tue Jan 23 09:49:12 2018 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Tue Jan 23 09:49:12 2018 +0900
----------------------------------------------------------------------
spark/spark-2.0/pom.xml | 30 +-----------------------------
spark/spark-2.1/pom.xml | 30 +-----------------------------
spark/spark-2.2/pom.xml | 31 ++-----------------------------
3 files changed, 4 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/afb2774e/spark/spark-2.0/pom.xml
----------------------------------------------------------------------
diff --git a/spark/spark-2.0/pom.xml b/spark/spark-2.0/pom.xml
index 3d82f89..9286697 100644
--- a/spark/spark-2.0/pom.xml
+++ b/spark/spark-2.0/pom.xml
@@ -27,7 +27,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>hivemall-spark_2.0</artifactId>
+ <artifactId>hivemall-spark2.0</artifactId>
<name>Hivemall on Spark 2.0</name>
<packaging>jar</packaging>
@@ -162,34 +162,6 @@
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>SparkTestSuite.txt</filereports>
- <argLine>${spark.test.jvm.opts}</argLine>
- <stderr />
- <environmentVariables>
- <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
- <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
- <SPARK_TESTING>1</SPARK_TESTING>
- <JAVA_HOME>${env.JAVA_HOME}</JAVA_HOME>
- <PATH>${env.JAVA_HOME}/bin:${env.PATH}</PATH>
- </environmentVariables>
- <systemProperties>
- <log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
- <derby.system.durability>test</derby.system.durability>
- <java.awt.headless>true</java.awt.headless>
- <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
- <spark.testing>1</spark.testing>
- <spark.ui.enabled>false</spark.ui.enabled>
- <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
- <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
- <!-- Needed by sql/hive tests. -->
- <test.src.tables>__not_used__</test.src.tables>
- </systemProperties>
- <tagsToExclude>${test.exclude.tags}</tagsToExclude>
- </configuration>
<executions>
<execution>
<id>test</id>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/afb2774e/spark/spark-2.1/pom.xml
----------------------------------------------------------------------
diff --git a/spark/spark-2.1/pom.xml b/spark/spark-2.1/pom.xml
index e1da779..003226e 100644
--- a/spark/spark-2.1/pom.xml
+++ b/spark/spark-2.1/pom.xml
@@ -27,7 +27,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>hivemall-spark_2.1</artifactId>
+ <artifactId>hivemall-spark2.1</artifactId>
<name>Hivemall on Spark 2.1</name>
<packaging>jar</packaging>
@@ -162,34 +162,6 @@
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>SparkTestSuite.txt</filereports>
- <argLine>${spark.test.jvm.opts}</argLine>
- <stderr />
- <environmentVariables>
- <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
- <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
- <SPARK_TESTING>1</SPARK_TESTING>
- <JAVA_HOME>${env.JAVA_HOME}</JAVA_HOME>
- <PATH>${env.JAVA_HOME}/bin:${env.PATH}</PATH>
- </environmentVariables>
- <systemProperties>
- <log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
- <derby.system.durability>test</derby.system.durability>
- <java.awt.headless>true</java.awt.headless>
- <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
- <spark.testing>1</spark.testing>
- <spark.ui.enabled>false</spark.ui.enabled>
- <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
- <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
- <!-- Needed by sql/hive tests. -->
- <test.src.tables>__not_used__</test.src.tables>
- </systemProperties>
- <tagsToExclude>${test.exclude.tags}</tagsToExclude>
- </configuration>
<executions>
<execution>
<id>test</id>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/afb2774e/spark/spark-2.2/pom.xml
----------------------------------------------------------------------
diff --git a/spark/spark-2.2/pom.xml b/spark/spark-2.2/pom.xml
index 1d9e3c8..9e96d80 100644
--- a/spark/spark-2.2/pom.xml
+++ b/spark/spark-2.2/pom.xml
@@ -27,7 +27,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>hivemall-spark_2.2</artifactId>
+ <artifactId>hivemall-spark2.2</artifactId>
<name>Hivemall on Spark 2.2</name>
<packaging>jar</packaging>
@@ -35,6 +35,7 @@
<main.basedir>${project.parent.parent.basedir}</main.basedir>
<spark.version>2.2.0</spark.version>
<spark.binary.version>2.2</spark.binary.version>
+ <scalatest.jvm.opts>-ea -Xms768m -Xmx2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=512m -XX:ReservedCodeCacheSize=512m</scalatest.jvm.opts>
<java.source.version>1.8</java.source.version>
<java.target.version>1.8</java.target.version>
<maven.compiler.source>1.8</maven.compiler.source>
@@ -166,34 +167,6 @@
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>SparkTestSuite.txt</filereports>
- <argLine>-ea -Xmx2g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=${CodeCacheSize}</argLine>
- <stderr />
- <environmentVariables>
- <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
- <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
- <SPARK_TESTING>1</SPARK_TESTING>
- <JAVA_HOME>${env.JAVA_HOME}</JAVA_HOME>
- <PATH>${env.JAVA_HOME}/bin:${env.PATH}</PATH>
- </environmentVariables>
- <systemProperties>
- <log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
- <derby.system.durability>test</derby.system.durability>
- <java.awt.headless>true</java.awt.headless>
- <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
- <spark.testing>1</spark.testing>
- <spark.ui.enabled>false</spark.ui.enabled>
- <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
- <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
- <!-- Needed by sql/hive tests. -->
- <test.src.tables>__not_used__</test.src.tables>
- </systemProperties>
- <tagsToExclude>${test.exclude.tags}</tagsToExclude>
- </configuration>
<executions>
<execution>
<id>test</id>
[3/3] incubator-hivemall git commit: Updated pom.xml
Posted by my...@apache.org.
Updated pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/26e3d705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/26e3d705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/26e3d705
Branch: refs/heads/v0.5.0
Commit: 26e3d705f993b03480ce98633c48cae01c34b5b8
Parents: afb2774
Author: Makoto Yui <my...@apache.org>
Authored: Tue Jan 23 09:49:40 2018 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Tue Jan 23 09:49:40 2018 +0900
----------------------------------------------------------------------
core/pom.xml | 5 -----
pom.xml | 43 ++-----------------------------------------
spark/pom.xml | 37 +++++++++++++++++++++++++++++++++++--
3 files changed, 37 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/26e3d705/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index c87ded1..160c8ed 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -195,11 +195,6 @@
<Implementation-Vendor>${project.organization.name}</Implementation-Vendor>
</manifestEntries>
</transformer>
- <!--
- <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- </transformer>
- -->
</transformers>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/26e3d705/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cc3bdce..c31e9f8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -269,7 +269,7 @@
<junit.version>4.12</junit.version>
<dependency.locations.enabled>false</dependency.locations.enabled>
<main.basedir>${project.basedir}</main.basedir>
- <maven-enforcer-plugin.version>3.0.0-M1</maven-enforcer-plugin.version>
+ <maven-enforcer-plugin.version>3.3.1</maven-enforcer-plugin.version>
<surefire.version>2.19.1</surefire.version>
</properties>
@@ -314,46 +314,6 @@
<profiles>
<profile>
- <id>spark-2.1</id>
- <modules>
- <module>spark/spark-2.1</module>
- <module>spark/spark-common</module>
- </modules>
- <properties>
- <spark.version>2.1.1</spark.version>
- <spark.binary.version>2.1</spark.binary.version>
- </properties>
- </profile>
- <profile>
- <id>spark-2.0</id>
- <modules>
- <module>spark/spark-2.0</module>
- <module>spark/spark-common</module>
- </modules>
- <properties>
- <spark.version>2.0.2</spark.version>
- <spark.binary.version>2.0</spark.binary.version>
- </properties>
- </profile>
- <profile>
- <id>java7</id>
- <properties>
- <spark.test.jvm.opts>-ea -Xms768m -Xmx1024m -XX:PermSize=128m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</spark.test.jvm.opts>
- </properties>
- <activation>
- <jdk>[,1.8)</jdk> <!-- version < 1.8 -->
- </activation>
- </profile>
- <profile>
- <id>java8</id>
- <properties>
- <spark.test.jvm.opts>-ea -Xms768m -Xmx1024m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=512m -XX:ReservedCodeCacheSize=512m</spark.test.jvm.opts>
- </properties>
- <activation>
- <jdk>[1.8,)</jdk> <!-- version >= 1.8 -->
- </activation>
- </profile>
- <profile>
<id>doclint-java8-disable</id>
<activation>
<jdk>[1.8,)</jdk>
@@ -482,6 +442,7 @@
<version>3.0.2</version>
<configuration>
<finalName>${project.artifactId}-${project.version}</finalName>
+ <outputDirectory>${main.basedir}</outputDirectory>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/26e3d705/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 7db85ab..fb85eef 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -42,6 +42,7 @@
<main.basedir>${project.parent.basedir}</main.basedir>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>
+ <scalatest.jvm.opts>-ea -Xms768m -Xmx1024m -XX:PermSize=128m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</scalatest.jvm.opts>
</properties>
<dependencyManagement>
@@ -109,6 +110,38 @@
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
</plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <filereports>SparkTestSuite.txt</filereports>
+ <argLine>${scalatest.jvm.opts}</argLine>
+ <stderr />
+ <environmentVariables>
+ <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
+ <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
+ <SPARK_TESTING>1</SPARK_TESTING>
+ <JAVA_HOME>${env.JAVA_HOME}</JAVA_HOME>
+ <PATH>${env.JAVA_HOME}/bin:${env.PATH}</PATH>
+ </environmentVariables>
+ <systemProperties>
+ <log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
+ <derby.system.durability>test</derby.system.durability>
+ <java.awt.headless>true</java.awt.headless>
+ <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
+ <spark.testing>1</spark.testing>
+ <spark.ui.enabled>false</spark.ui.enabled>
+ <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
+ <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
+ <!-- Needed by sql/hive tests. -->
+ <test.src.tables>__not_used__</test.src.tables>
+ </systemProperties>
+ <tagsToExclude>${test.exclude.tags}</tagsToExclude>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
@@ -145,9 +178,9 @@
<!-- <arg>-feature</arg> -->
</args>
<jvmArgs>
- <jvmArg>-Xms512m</jvmArg>
+ <jvmArg>-Xms768m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
- <jvmArg>-XX:PermSize=64m</jvmArg>
+ <jvmArg>-XX:PermSize=128m</jvmArg>
<jvmArg>-XX:MaxPermSize=512m</jvmArg>
<jvmArg>-XX:ReservedCodeCacheSize=512m</jvmArg>
</jvmArgs>