You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mnemonic.apache.org by ga...@apache.org on 2017/04/18 21:20:48 UTC
incubator-mnemonic git commit: MNEMONIC-236: Construct a workable
skeleton sub-project of mnemonic-spark
Repository: incubator-mnemonic
Updated Branches:
refs/heads/master c7ce58c21 -> 11fd621df
MNEMONIC-236: Construct a workable skeleton sub-project of mnemonic-spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/11fd621d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/11fd621d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/11fd621d
Branch: refs/heads/master
Commit: 11fd621dff1d1a555892fd9c58af25277a1cd740
Parents: c7ce58c
Author: Wang, Gang(Gary) <ga...@intel.com>
Authored: Tue Apr 18 14:20:00 2017 -0700
Committer: Wang, Gang(Gary) <ga...@intel.com>
Committed: Tue Apr 18 14:20:00 2017 -0700
----------------------------------------------------------------------
.gitignore | 2 +
build-tools/test.conf | 4 +
mnemonic-spark/mnemonic-spark-core/pom.xml | 84 +++++++
.../src/main/resources/log4j.properties | 33 +++
.../apache/mnemonic/spark/rdd/DurableRDD.scala | 22 ++
.../src/test/resources/testng.xml | 34 +++
.../mnemonic/spark/rdd/DurableRDDSuite.scala | 35 +++
mnemonic-spark/pom.xml | 223 +++++++++++++++++++
pom.xml | 6 +-
9 files changed, 441 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index ad45fab..182beb7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,4 +44,6 @@ testlog/
*.patch
*.swp
*.pyc
+.cache-tests
+.cache-main
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/build-tools/test.conf
----------------------------------------------------------------------
diff --git a/build-tools/test.conf b/build-tools/test.conf
index 93beb92..ccf997f 100644
--- a/build-tools/test.conf
+++ b/build-tools/test.conf
@@ -56,3 +56,7 @@ mvn -Dtest=MneMapreduceChunkDataTest test -pl mnemonic-hadoop/mnemonic-hadoop-ma
# a testcase for module "mnemonic-hadoop/mnemonic-hadoop-mapreduce" that requires 'pmalloc' memory service to pass
mvn -Dtest=MneMapredBufferDataTest test -pl mnemonic-hadoop/mnemonic-hadoop-mapreduce -DskipTests=false
+
+# a testcase for module "mnemonic-spark/mnemonic-spark-core" that requires 'pmalloc' memory service to pass
+mvn -Dsuites=org.apache.mnemonic.spark.rdd.DurableRDDSuite test -pl mnemonic-spark/mnemonic-spark-core -DskipTests=false
+
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/mnemonic-spark-core/pom.xml
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/pom.xml b/mnemonic-spark/mnemonic-spark-core/pom.xml
new file mode 100644
index 0000000..4440083
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.mnemonic</groupId>
+ <artifactId>mnemonic-spark</artifactId>
+ <version>0.6.0-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>mnemonic-spark-core</artifactId>
+ <name>mnemonic-spark-core</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.bsc.maven</groupId>
+ <artifactId>maven-processor-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/mnemonic-spark-core/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/resources/log4j.properties b/mnemonic-spark/mnemonic-spark-core/src/main/resources/log4j.properties
new file mode 100644
index 0000000..24b0e3c
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/resources/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Root logger option
+log4j.rootLogger=INFO, file, stdout
+
+#to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+#to log.log file
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File=log.log
+log4j.appender.file.MaxFileSize=10MB
+log4j.appender.file.MaxBackupIndex=10
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
new file mode 100644
index 0000000..164033e
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark.rdd;
+
+class DurableRDD {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/mnemonic-spark-core/src/test/resources/testng.xml
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/resources/testng.xml b/mnemonic-spark/mnemonic-spark-core/src/test/resources/testng.xml
new file mode 100644
index 0000000..3dcc372
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/resources/testng.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" verbose="1" parallel="tests" thread-count="1">
+ <test name="Test">
+ <classes>
+ <class name="org.apache.mnemonic.mapreduce"/>
+ </classes>
+ </test> <!-- Test -->
+</suite> <!-- Suite -->
+
+
+<!--
+ <class name="org.apache.mnemonic.service.computingservice.DurableSinglyLinkedListNGPrintTest"/>
+ -->
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
new file mode 100644
index 0000000..e59aad7
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark.rdd;
+
+import org.scalatest.FlatSpec
+import org.scalatest.Matchers
+import org.scalatest.BeforeAndAfter
+
+class DurableRDDSuite extends FlatSpec with Matchers with BeforeAndAfter {
+
+ "An empty Set" should "have size 0" in {
+ assert(Set.empty.size == 0)
+ }
+
+ it should "produce NoSuchElementException when head is invoked" in {
+ assertThrows[NoSuchElementException] {
+ Set.empty.head
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/mnemonic-spark/pom.xml
----------------------------------------------------------------------
diff --git a/mnemonic-spark/pom.xml b/mnemonic-spark/pom.xml
new file mode 100644
index 0000000..76beef8
--- /dev/null
+++ b/mnemonic-spark/pom.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.mnemonic</groupId>
+ <artifactId>mnemonic-parent</artifactId>
+ <version>0.6.0-incubating-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>mnemonic-spark</artifactId>
+ <name>mnemonic-spark</name>
+ <packaging>pom</packaging>
+ <url>http://mnemonic.incubator.apache.org</url>
+
+ <properties>
+ <spark.version>2.1.0</spark.version>
+ <scala.version>2.11.8</scala.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scalatest.version>3.0.1</scalatest.version>
+ <spark-scope>compile</spark-scope>
+ </properties>
+
+ <modules>
+ <module>mnemonic-spark-core</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.mnemonic</groupId>
+ <artifactId>mnemonic-core</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mnemonic</groupId>
+ <artifactId>mnemonic-collections</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mnemonic</groupId>
+ <artifactId>mnemonic-sessions</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.flowcomputing.commons</groupId>
+ <artifactId>commons-primitives</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-mllib_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <version>${scala.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ <version>${scala.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-actors</artifactId>
+ <version>${scala.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scalap</artifactId>
+ <version>${scala.version}</version>
+ <scope>${spark-scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scalactic</groupId>
+ <artifactId>scalactic_${scala.binary.version}</artifactId>
+ <version>${scalatest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <version>${scalatest.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- logging dependencies -->
+ <!-- assume all APIs will be used -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <filereports>SparkIntegrationTestSuite.txt</filereports>
+ <skipTests>${skipTests}</skipTests>
+ </configuration>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <forceCreation>true</forceCreation>
+ <archive>
+ <addMavenDescriptor>false</addMavenDescriptor>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <recompileMode>incremental</recompileMode>
+ <useZincServer>true</useZincServer>
+ <args>
+ <arg>-unchecked</arg>
+ <arg>-deprecation</arg>
+ <arg>-feature</arg>
+ <arg>-language:postfixOps</arg>
+ </args>
+ <jvmArgs>
+ <jvmArg>-Xms1024m</jvmArg>
+ <jvmArg>-Xmx1024m</jvmArg>
+ </jvmArgs>
+ <javacArgs>
+ <javacArg>-source</javacArg>
+ <javacArg>${java.version}</javacArg>
+ <javacArg>-target</javacArg>
+ <javacArg>${java.version}</javacArg>
+ </javacArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/11fd621d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3f23d07..ad101f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -95,9 +95,11 @@
<module>mnemonic-benches</module>
<module>mnemonic-sessions</module>
<module>mnemonic-hadoop</module>
+ <module>mnemonic-spark</module>
</modules>
<properties>
+ <java.version>1.7</java.version>
<maven.min-version>3.2.1</maven.min-version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -322,8 +324,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
<compilerArgs>
<arg>-XDignore.symbol.file</arg>
<arg>-XDenableSunApiLintControl</arg>