You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2018/08/08 14:47:22 UTC
datafu git commit: Skeleton spark subproject setup
Repository: datafu
Updated Branches:
refs/heads/spark-tmp [created] 157d90819
Skeleton spark subproject setup
Project: http://git-wip-us.apache.org/repos/asf/datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/datafu/commit/157d9081
Tree: http://git-wip-us.apache.org/repos/asf/datafu/tree/157d9081
Diff: http://git-wip-us.apache.org/repos/asf/datafu/diff/157d9081
Branch: refs/heads/spark-tmp
Commit: 157d908196188ab296de9d83dd8fda9912b413f1
Parents: 1e69bf1
Author: Eyal Allweil <ey...@apache.org>
Authored: Wed Aug 8 15:58:03 2018 +0300
Committer: Eyal Allweil <ey...@apache.org>
Committed: Wed Aug 8 15:58:03 2018 +0300
----------------------------------------------------------------------
build.gradle | 3 +-
datafu-spark/.gitignore | 6 +
datafu-spark/build.gradle | 180 +++++++++
.../dependency-versions-scala-2.10.gradle | 29 ++
.../dependency-versions-scala-2.11.gradle | 31 ++
.../dependency-versions-scala-2.12.gradle | 29 ++
.../src/main/resources/META-INF/DISCLAIMER | 1 +
.../src/main/resources/META-INF/LICENSE | 393 +++++++++++++++++++
datafu-spark/src/main/resources/META-INF/NOTICE | 60 +++
gradle.properties | 3 +-
settings.gradle | 2 +-
11 files changed, 734 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 4b7f396..d26b980 100644
--- a/build.gradle
+++ b/build.gradle
@@ -92,7 +92,8 @@ rat {
'datafu-pig/input*',
'datafu-pig/docs',
'datafu-pig/queries',
- 'datafu-pig/query'
+ 'datafu-pig/query',
+ 'datafu-spark/derby.log'
]
}
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/.gitignore
----------------------------------------------------------------------
diff --git a/datafu-spark/.gitignore b/datafu-spark/.gitignore
new file mode 100644
index 0000000..6f74b9d
--- /dev/null
+++ b/datafu-spark/.gitignore
@@ -0,0 +1,6 @@
+/bin
+/test-output
+/data
+/.apt_generated/
+/out
+derby.log
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/build.gradle b/datafu-spark/build.gradle
new file mode 100644
index 0000000..6ca209c
--- /dev/null
+++ b/datafu-spark/build.gradle
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+buildscript {
+ repositories {
+ mavenCentral()
+ }
+}
+
+plugins {
+ id "de.undercouch.download" version "3.3.0"
+}
+
+apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")
+
+apply plugin: 'scala'
+apply plugin: datafu.autojar.GradleAutojarPlugin
+
+// allprojects is from samza
+allprojects {
+ // For all scala compilation, add extra compiler options, taken from version-specific
+ // dependency-versions-scala file applied above.
+ tasks.withType(ScalaCompile) {
+ scalaCompileOptions.additionalParameters = [ scalaOptions ]
+ }
+}
+
+archivesBaseName = 'datafu-spark_' + scalaVersion
+
+import groovy.xml.MarkupBuilder
+
+// the autojarred configuration includes all JARs that will be included
+// in the final JAR via autojar
+configurations.create('autojarred')
+
+configurations.create('jarjar')
+
+configurations {
+ compile {
+ extendsFrom autojarred
+ }
+}
+
+cleanEclipse {
+ doLast {
+ delete ".apt_generated"
+ delete ".settings"
+ delete ".factorypath"
+ delete "bin"
+ }
+}
+
+jar
+{
+ // initial jar only includes the main classes of datafu, not the dependencies.
+ // this is not the one we'll publish.
+ classifier = "core"
+}
+
+ext
+{
+ autojarBuildDir = tasks.jar.destinationDir
+}
+
+task jarWithDependencies(type: Autojar) {
+ description 'Creates a jar that includes the dependencies (under their own namespaces)'
+ autojarFiles = [
+ tasks.jar.getArchivePath().absoluteFile,
+ "META-INF/LICENSE",
+ "META-INF/DISCLAIMER",
+ "META-INF/NOTICE"
+ ]
+ targetConfiguration = configurations.autojarred
+ autojarExtra = '-baeq'
+}
+
+def outputFile = file(tasks.jar.getArchivePath().absoluteFile.toString().replace("-core","-jarjar"))
+
+task jarWithDependenciesNamespaced(dependsOn: jarWithDependencies) {
+ description 'Creates the jar that includes dependencies (under a datafu namespace)'
+
+ doLast {
+ project.ant {
+ taskdef name: "jarjar", classname: "com.tonicsystems.jarjar.JarJarTask", classpath: configurations.jarjar.asPath
+ jarjar(jarfile: outputFile, filesetmanifest: "merge") {
+ zipfileset(src: tasks.jarWithDependencies.autojarOutput)
+ rule pattern: "org.apache.commons.math.**", result: "datafu.org.apache.commons.math.@1"
+ rule pattern: "com.google.common.**", result: "datafu.com.google.common.@1"
+ }
+ }
+ }
+}
+
+task finalJar(type: Jar, dependsOn: jarWithDependenciesNamespaced) {
+ description 'Creates the final jar'
+
+ from(zipTree(outputFile))
+}
+
+// don't publish the core archive, as this doesn't have the dependencies
+configurations.archives.artifacts.removeAll { return it.classifier == "core"; }
+
+artifacts {
+ archives finalJar
+}
+
+// forcing scala joint compilation is from samza -------------------------
+
+// Force scala joint compilation
+sourceSets.main.scala.srcDir "src/main/java"
+sourceSets.test.scala.srcDir "src/test/java"
+
+// Disable the Javac compiler by forcing joint compilation by scalac. This is equivalent to setting
+// tasks.compileTestJava.enabled = false
+sourceSets.main.java.srcDirs = []
+sourceSets.test.java.srcDirs = []
+
+dependencies {
+ // all these dependencies are taken from samza
+ compile "org.scala-lang:scala-library:$scalaLibVersion"
+
+ // dependencies that are packaged into the jar using autojar
+ // autojar only includes what is needed
+ autojarred "org.apache.commons:commons-math:$commonsMathVersion"
+ autojarred "com.google.guava:guava:$guavaVersion"
+
+ // needed to run jarjar
+ jarjar "com.googlecode.jarjar:jarjar:1.3"
+
+ testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.2.0_0.10.0"
+ testCompile "org.scalatest:scalatest_" + scalaVersion + ":2.2.0"
+}
+
+modifyPom {
+ project {
+ dependencies {
+ // No dependencies because everything we need is autojarred.
+ }
+ }
+}
+
+// we need to set up the build for hadoop 3
+if (hadoopVersion.startsWith("2.")) {
+ dependencies {
+ // needed for compilation only. obviously don't need to autojar this.
+ compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
+ compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
+ compile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
+ compile "org.apache.spark:spark-core_" + scalaVersion + ":2.2.0"
+ compile "org.apache.spark:spark-hive_" + scalaVersion + ":2.2.0"
+ }
+} else {
+ dependencies {
+ // needed for compilation only. obviously don't need to autojar this.
+ compile "org.apache.hadoop:hadoop-core:$hadoopVersion"
+ }
+}
+
+test {
+ systemProperty 'datafu.jar.dir', file('build/libs')
+ systemProperty 'datafu.data.dir', file('data')
+
+ maxHeapSize = "2G"
+}
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.10.gradle b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
new file mode 100644
index 0000000..d128886
--- /dev/null
+++ b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ext {
+ scalaVersion = "2.10"
+ scalaLibVersion = "2.10.4"
+ // Extra options for the compiler:
+ // -feature: Give detailed warnings about language feature use (rather than just 'there were 4 warnings')
+ // -language:implicitConversions: Allow the use of implicit conversions without warning or library import
+ // -language:reflectiveCalls: Allow the automatic use of reflection to access fields without warning or library import
+// scalaOptions = "-feature -language:implicitConversions -language:reflectiveCalls"
+ scalatraVersion = "2.2.1"
+ jettyVersion = "9.2.7.v20150116"
+}
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.11.gradle b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
new file mode 100644
index 0000000..5306df0
--- /dev/null
+++ b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ext {
+ scalaVersion = "2.11"
+ scalaLibVersion = "2.11.8"
+ // Extra options for the compiler:
+ // -feature: Give detailed warnings about language feature use (rather than just 'there were 4 warnings')
+ // -language:implicitConversions: Allow the use of implicit conversions without warning or library import
+ // -language:reflectiveCalls: Allow the automatic use of reflection to access fields without warning or library import
+ // scalaOptions = "-feature -language:implicitConversions -language:reflectiveCalls"
+ scalaOptions = "-language:implicitConversions"
+
+ scalatraVersion = "2.5.0"
+ jettyVersion = "9.2.7.v20150116"
+}
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.12.gradle b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
new file mode 100644
index 0000000..fc8c6cd
--- /dev/null
+++ b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ext {
+ scalaVersion = "2.12"
+ scalaLibVersion = "2.12.1"
+ // Extra options for the compiler:
+ // -feature: Give detailed warnings about language feature use (rather than just 'there were 4 warnings')
+ // -language:implicitConversions: Allow the use of implicit conversions without warning or library import
+ // -language:reflectiveCalls: Allow the automatic use of reflection to access fields without warning or library import
+ scalaOptions = "-feature -language:implicitConversions -language:reflectiveCalls"
+ scalatraVersion = "2.5.0"
+ jettyVersion = "9.2.7.v20150116"
+}
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/src/main/resources/META-INF/DISCLAIMER
----------------------------------------------------------------------
diff --git a/datafu-spark/src/main/resources/META-INF/DISCLAIMER b/datafu-spark/src/main/resources/META-INF/DISCLAIMER
new file mode 100644
index 0000000..0c43c4c
--- /dev/null
+++ b/datafu-spark/src/main/resources/META-INF/DISCLAIMER
@@ -0,0 +1 @@
+Apache DataFu is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/datafu-spark/src/main/resources/META-INF/LICENSE b/datafu-spark/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..6bd634b
--- /dev/null
+++ b/datafu-spark/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,393 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+=======================================================================
+
+APACHE DATAFU-PIG SUBCOMPONENTS:
+
+The Apache DataFu datafu-pig-incubating JAR bundles the following
+Apache-2.0-licensed dependencies:
+
+* it.unimi.dsi:fastutil:6.5.7
+* org.apache.commons:commons-math:2.2
+* com.clearspring.analytics:stream:2.5.0
+* com.google.guava:guava:11.0.2
+* org.apache.opennlp:opennlp-tools:1.5.3
+* org.apache.opennlp:opennlp-uima:1.5.3
+* org.apache.opennlp:opennlp-maxent:3.0.3
+
+
+Contents copied from LICENSE.txt for org.apache.commons:commons-math:2.2:
+
+=======================================================================
+
+APACHE COMMONS MATH DERIVATIVE WORKS:
+
+The Apache commons-math library includes a number of subcomponents
+whose implementation is derived from original sources written
+in C or Fortran. License terms of the original sources
+are reproduced below.
+
+===============================================================================
+For the lmder, lmpar and qrsolv Fortran routine from minpack and translated in
+the LevenbergMarquardtOptimizer class in package
+org.apache.commons.math.optimization.general
+Original source copyright and license statement:
+
+Minpack Copyright Notice (1999) University of Chicago. All rights reserved
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions of source code must retain the above
+copyright notice, this list of conditions and the following
+disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials
+provided with the distribution.
+
+3. The end-user documentation included with the
+redistribution, if any, must include the following
+acknowledgment:
+
+ "This product includes software developed by the
+ University of Chicago, as Operator of Argonne National
+ Laboratory.
+
+Alternately, this acknowledgment may appear in the software
+itself, if and wherever such third-party acknowledgments
+normally appear.
+
+4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
+WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
+UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
+THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
+OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
+OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
+USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
+THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
+DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
+UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
+BE CORRECTED.
+
+5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
+HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
+ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
+INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
+ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
+PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
+SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
+(INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
+EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
+POSSIBILITY OF SUCH LOSS OR DAMAGES.
+===============================================================================
+
+Copyright and license statement for the odex Fortran routine developed by
+E. Hairer and G. Wanner and translated in GraggBulirschStoerIntegrator class
+in package org.apache.commons.math.ode.nonstiff:
+
+
+Copyright (c) 2004, Ernst Hairer
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original lapack fortran routines
+translated in EigenDecompositionImpl class in package
+org.apache.commons.math.linear:
+
+Copyright (c) 1992-2008 The University of Tennessee. All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer listed
+ in this license in the documentation and/or other materials
+ provided with the distribution.
+
+- Neither the name of the copyright holders nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+===============================================================================
+
+Copyright and license statement for the original Mersenne twister C
+routines translated in MersenneTwister class in package
+org.apache.commons.math.random:
+
+ Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. The names of its contributors may not be used to endorse or promote
+ products derived from this software without specific prior written
+ permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/datafu-spark/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/datafu-spark/src/main/resources/META-INF/NOTICE b/datafu-spark/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..18fba06
--- /dev/null
+++ b/datafu-spark/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,60 @@
+Apache DataFu
+Copyright 2010-2017 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+Contents copied from NOTICE.txt for org.apache.commons:commons-math:2.2:
+
+===============================================================================
+
+The BracketFinder (package org.apache.commons.math.optimization.univariate)
+and PowellOptimizer (package org.apache.commons.math.optimization.general)
+classes are based on the Python code in module "optimize.py" (version 0.5)
+developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/)
+Copyright © 2003-2009 SciPy Developers.
+===============================================================================
+
+The LinearConstraint, LinearObjectiveFunction, LinearOptimizer,
+RelationShip, SimplexSolver and SimplexTableau classes in package
+org.apache.commons.math.optimization.linear include software developed by
+Benjamin McCann (http://www.benmccann.com) and distributed with
+the following copyright: Copyright 2009 Google Inc.
+===============================================================================
+
+This product includes software developed by the
+University of Chicago, as Operator of Argonne National
+Laboratory.
+The LevenbergMarquardtOptimizer class in package
+org.apache.commons.math.optimization.general includes software
+translated from the lmder, lmpar and qrsolv Fortran routines
+from the Minpack package
+Minpack Copyright Notice (1999) University of Chicago. All rights reserved
+===============================================================================
+
+The GraggBulirschStoerIntegrator class in package
+org.apache.commons.math.ode.nonstiff includes software translated
+from the odex Fortran routine developed by E. Hairer and G. Wanner.
+Original source copyright:
+Copyright (c) 2004, Ernst Hairer
+===============================================================================
+
+The EigenDecompositionImpl class in package
+org.apache.commons.math.linear includes software translated
+from some LAPACK Fortran routines. Original source copyright:
+Copyright (c) 1992-2008 The University of Tennessee. All rights reserved.
+===============================================================================
+
+The MersenneTwister class in package org.apache.commons.math.random
+includes software translated from the 2002-01-26 version of
+the Mersenne-Twister generator written in C by Makoto Matsumoto and Takuji
+Nishimura. Original source copyright:
+Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+All rights reserved
+===============================================================================
+
+The complete text of licenses and disclaimers associated with the the original
+sources enumerated above at the time of code translation are in the
+LICENSE.txt [copied into LICENSE for Apache DataFu] file.
+
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/gradle.properties
----------------------------------------------------------------------
diff --git a/gradle.properties b/gradle.properties
index aa953be..10ce0cf 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -19,4 +19,5 @@ group=org.apache.datafu
version=1.4.0
gradleVersion=4.8.1
org.gradle.jvmargs="-XX:MaxPermSize=512m"
-release=false
\ No newline at end of file
+scalaVersion=2.11
+release=false
http://git-wip-us.apache.org/repos/asf/datafu/blob/157d9081/settings.gradle
----------------------------------------------------------------------
diff --git a/settings.gradle b/settings.gradle
index 7556761..8c478a6 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -17,4 +17,4 @@
* under the License.
*/
-include "build-plugin","datafu-pig","datafu-hourglass"
\ No newline at end of file
+include "build-plugin","datafu-pig","datafu-hourglass","datafu-spark"