You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2012/07/25 21:15:25 UTC
[1/13] git commit: Update license for related projects + removed code
using LGPL
Updated Branches:
refs/heads/S4-88 [created] 96eddf879
Update license for related projects + removed code using LGPL
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/96eddf87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/96eddf87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/96eddf87
Branch: refs/heads/S4-88
Commit: 96eddf879f11c1d2fceb3705d4072663c10c63b3
Parents: 94f54a1
Author: Matthieu Morel <mm...@apache.org>
Authored: Wed Jul 25 22:53:47 2012 +0200
Committer: Matthieu Morel <mm...@apache.org>
Committed: Wed Jul 25 23:09:16 2012 +0200
----------------------------------------------------------------------
.gitignore | 3 +
LICENSE | 281 +++++++++++-
NOTICE.txt | 12 +
README.md | 6 +-
build.gradle | 6 +-
subprojects/s4-example/s4-example.gradle | 31 +-
.../apache/s4/example/model/ClassIDKeyFinder.java | 39 --
.../org/apache/s4/example/model/Controller.java | 246 ---------
.../java/org/apache/s4/example/model/DataFile.java | 65 ---
.../apache/s4/example/model/HypIDKeyFinder.java | 39 --
.../java/org/apache/s4/example/model/Main.java | 34 --
.../org/apache/s4/example/model/MaximizerPE.java | 99 ----
.../org/apache/s4/example/model/MetricsPE.java | 150 ------
.../java/org/apache/s4/example/model/ModelPE.java | 225 ---------
.../java/org/apache/s4/example/model/Module.java | 80 ---
.../java/org/apache/s4/example/model/MyApp.java | 174 -------
.../java/org/apache/s4/example/model/ObsEvent.java | 102 ----
.../apache/s4/example/model/ObsIndexKeyFinder.java | 39 --
.../java/org/apache/s4/example/model/README.md | 390 ---------------
.../org/apache/s4/example/model/ResultEvent.java | 60 ---
.../apache/s4/example/model/ResultKeyFinder.java | 39 --
.../main/java/org/apache/s4/model/Gaussian.java | 34 --
.../java/org/apache/s4/model/GaussianMixture.java | 34 --
.../org/apache/s4/model/GaussianMixtureModel.java | 320 ------------
.../java/org/apache/s4/model/GaussianModel.java | 383 --------------
.../src/main/java/org/apache/s4/model/Model.java | 97 ----
.../main/java/org/apache/s4/util/MatrixOps.java | 285 -----------
.../src/main/resources/covtype-test.data.gz | Bin 1646680 -> 0 bytes
.../src/main/resources/covtype-train-1000.data.gz | Bin 17321 -> 0 bytes
.../src/main/resources/covtype-train.data.gz | Bin 7918423 -> 0 bytes
.../s4-example/src/main/resources/model.properties | 14 -
.../src/main/resources/s4-piper-example.properties | 7 -
test-apps/simple-deployable-app-2/build.gradle | 245 ---------
.../src/main/java/org/apache/s4/deploy/A.java | 34 --
.../java/org/apache/s4/deploy/AppConstants.java | 28 -
.../main/java/org/apache/s4/deploy/TestApp.java | 56 --
36 files changed, 321 insertions(+), 3336 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 941182e..a13ec8f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,6 @@ pepe/
# Java
*.class
+
+# apache RAT
+.rat-excludes
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index f10b48c..fbb4f12 100644
--- a/LICENSE
+++ b/LICENSE
@@ -200,5 +200,282 @@
See the License for the specific language governing permissions and
limitations under the License.
-
-
\ No newline at end of file
+----------------------------------------------------------------------------
+
+This product uses Kryo, which uses the following license:
+
+Copyright (c) 2008, Nathan Sweet All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer. * Redistributions in binary
+form must reproduce the above copyright notice, this list of conditions and the
+following disclaimer in the documentation and/or other materials provided with
+the distribution. * Neither the name of Esoteric Software nor the names of its
+contributors may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------------
+
+This product uses Logback, which uses the following license (EPL1.0):
+
+Logback: the reliable, generic, fast and flexible logging framework. Copyright
+(C) 1999-2012, QOS.ch. All rights reserved.
+
+This program and the accompanying materials are dual-licensed under either the
+terms of the Eclipse Public License v1.0 as published by the Eclipse Foundation
+
+or (per the licensee's choosing)
+
+under the terms of the GNU Lesser General Public License version 2.1 as
+published by the Free Software Foundation.
+
+------------- Here is the corresponding EPL 1.0 license:
+
+Eclipse Public License - v 1.0 THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE
+TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR
+DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT�S ACCEPTANCE OF THIS
+AGREEMENT.
+
+1. DEFINITIONS
+
+"Contribution" means:
+
+a) in the case of the initial Contributor, the initial code and documentation
+distributed under this Agreement, and b) in the case of each subsequent
+Contributor:
+
+i)changes to the Program, and
+
+ii)additions to the Program;
+
+where such changes and/or additions to the Program originate from and are
+distributed by that particular Contributor. A Contribution 'originates' from a
+Contributor if it was added to the Program by such Contributor itself or anyone
+acting on such Contributor�s behalf. Contributions do not include additions to
+the Program which: (i) are separate modules of software distributed in
+conjunction with the Program under their own license agreement, and (ii) are not
+derivative works of the Program.
+
+"Contributor" means any person or entity that distributes the Program.
+
+"Licensed Patents " mean patent claims licensable by a Contributor which are
+necessarily infringed by the use or sale of its Contribution alone or when
+combined with the Program.
+
+"Program" means the Contributions distributed in accordance with this Agreement.
+
+"Recipient" means anyone who receives the Program under this Agreement,
+including all Contributors.
+
+2. GRANT OF RIGHTS
+
+a) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free copyright license to
+reproduce, prepare derivative works of, publicly display, publicly perform,
+distribute and sublicense the Contribution of such Contributor, if any, and such
+derivative works, in source code and object code form.
+
+b) Subject to the terms of this Agreement, each Contributor hereby grants
+Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed
+Patents to make, use, sell, offer to sell, import and otherwise transfer the
+Contribution of such Contributor, if any, in source code and object code form.
+This patent license shall apply to the combination of the Contribution and the
+Program if, at the time the Contribution is added by the Contributor, such
+addition of the Contribution causes such combination to be covered by the
+Licensed Patents. The patent license shall not apply to any other combinations
+which include the Contribution. No hardware per se is licensed hereunder.
+
+c) Recipient understands that although each Contributor grants the licenses to
+its Contributions set forth herein, no assurances are provided by any
+Contributor that the Program does not infringe the patent or other intellectual
+property rights of any other entity. Each Contributor disclaims any liability to
+Recipient for claims brought by any other entity based on infringement of
+intellectual property rights or otherwise. As a condition to exercising the
+rights and licenses granted hereunder, each Recipient hereby assumes sole
+responsibility to secure any other intellectual property rights needed, if any.
+For example, if a third party patent license is required to allow Recipient to
+distribute the Program, it is Recipient�s responsibility to acquire that license
+before distributing the Program.
+
+d) Each Contributor represents that to its knowledge it has sufficient copyright
+rights in its Contribution, if any, to grant the copyright license set forth in
+this Agreement.
+
+3. REQUIREMENTS
+
+A Contributor may choose to distribute the Program in object code form under its
+own license agreement, provided that:
+
+a) it complies with the terms and conditions of this Agreement; and
+
+b) its license agreement:
+
+i) effectively disclaims on behalf of all Contributors all warranties and
+conditions, express and implied, including warranties or conditions of title and
+non-infringement, and implied warranties or conditions of merchantability and
+fitness for a particular purpose;
+
+ii) effectively excludes on behalf of all Contributors all liability for
+damages, including direct, indirect, special, incidental and consequential
+damages, such as lost profits;
+
+iii) states that any provisions which differ from this Agreement are offered by
+that Contributor alone and not by any other party; and
+
+iv) states that source code for the Program is available from such Contributor,
+and informs licensees how to obtain it in a reasonable manner on or through a
+medium customarily used for software exchange.
+
+When the Program is made available in source code form:
+
+a) it must be made available under this Agreement; and
+
+b) a copy of this Agreement must be included with each copy of the Program.
+
+Contributors may not remove or alter any copyright notices contained within the
+Program.
+
+Each Contributor must identify itself as the originator of its Contribution, if
+any, in a manner that reasonably allows subsequent Recipients to identify the
+originator of the Contribution.
+
+4. COMMERCIAL DISTRIBUTION
+
+Commercial distributors of software may accept certain responsibilities with
+respect to end users, business partners and the like. While this license is
+intended to facilitate the commercial use of the Program, the Contributor who
+includes the Program in a commercial product offering should do so in a manner
+which does not create potential liability for other Contributors. Therefore, if
+a Contributor includes the Program in a commercial product offering, such
+Contributor ("Commercial Contributor") hereby agrees to defend and indemnify
+every other Contributor ("Indemnified Contributor") against any losses, damages
+and costs (collectively "Losses") arising from claims, lawsuits and other legal
+actions brought by a third party against the Indemnified Contributor to the
+extent caused by the acts or omissions of such Commercial Contributor in
+connection with its distribution of the Program in a commercial product
+offering. The obligations in this section do not apply to any claims or Losses
+relating to any actual or alleged intellectual property infringement. In order
+to qualify, an Indemnified Contributor must: a) promptly notify the Commercial
+Contributor in writing of such claim, and b) allow the Commercial Contributor to
+control, and cooperate with the Commercial Contributor in, the defense and any
+related settlement negotiations. The Indemnified Contributor may participate in
+any such claim at its own expense.
+
+For example, a Contributor might include the Program in a commercial product
+offering, Product X. That Contributor is then a Commercial Contributor. If that
+Commercial Contributor then makes performance claims, or offers warranties
+related to Product X, those performance claims and warranties are such
+Commercial Contributor�s responsibility alone. Under this section, the
+Commercial Contributor would have to defend claims against the other
+Contributors related to those performance claims and warranties, and if a court
+requires any other Contributor to pay any damages as a result, the Commercial
+Contributor must pay those damages.
+
+5. NO WARRANTY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR
+IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each
+Recipient is solely responsible for determining the appropriateness of using and
+distributing the Program and assumes all risks associated with its exercise of
+rights under this Agreement , including but not limited to the risks and costs
+of program errors, compliance with applicable laws, damage to or loss of data,
+programs or equipment, and unavailability or interruption of operations.
+
+6. DISCLAIMER OF LIABILITY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
+CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST
+PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS
+GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. GENERAL
+
+If any provision of this Agreement is invalid or unenforceable under applicable
+law, it shall not affect the validity or enforceability of the remainder of the
+terms of this Agreement, and without further action by the parties hereto, such
+provision shall be reformed to the minimum extent necessary to make such
+provision valid and enforceable.
+
+If Recipient institutes patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Program itself
+(excluding combinations of the Program with other software or hardware)
+infringes such Recipient�s patent(s), then such Recipient�s rights granted under
+Section 2(b) shall terminate as of the date such litigation is filed.
+
+All Recipient�s rights under this Agreement shall terminate if it fails to
+comply with any of the material terms or conditions of this Agreement and does
+not cure such failure in a reasonable period of time after becoming aware of
+such noncompliance. If all Recipient�s rights under this Agreement terminate,
+Recipient agrees to cease use and distribution of the Program as soon as
+reasonably practicable. However, Recipient�s obligations under this Agreement
+and any licenses granted by Recipient relating to the Program shall continue and
+survive.
+
+Everyone is permitted to copy and distribute copies of this Agreement, but in
+order to avoid inconsistency the Agreement is copyrighted and may only be
+modified in the following manner. The Agreement Steward reserves the right to
+publish new versions (including revisions) of this Agreement from time to time.
+No one other than the Agreement Steward has the right to modify this Agreement.
+The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation
+may assign the responsibility to serve as the Agreement Steward to a suitable
+separate entity. Each new version of the Agreement will be given a
+distinguishing version number. The Program (including Contributions) may always
+be distributed subject to the version of the Agreement under which it was
+received. In addition, after a new version of the Agreement is published,
+Contributor may elect to distribute the Program (including its Contributions)
+under the new version. Except as expressly stated in Sections 2(a) and 2(b)
+above, Recipient receives no rights or licenses to the intellectual property of
+any Contributor under this Agreement, whether expressly, by implication,
+estoppel or otherwise. All rights in the Program not expressly granted under
+this Agreement are reserved.
+
+This Agreement is governed by the laws of the State of New York and the
+intellectual property laws of the United States of America. No party to this
+Agreement will bring a legal action under this Agreement more than one year
+after the cause of action arose. Each party waives its rights to a jury trial in
+any resulting litigation.
+
+
+
+----------------------------------------------------------------------------
+
+Copyright (c) 2004-2011 QOS.ch
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 578e5d5..63abaeb 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -3,3 +3,15 @@ Copyright 2012 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+
+This product includes kryo, developed by Nathan Sweet
+(http://www.kryo.googlecode.com)
+
+This product include software developed in the netty project
+in binary form (http://netty.io)
+
+This product includes logback, developed by QOS.ch, in binary form
+(http://logback.qos.ch/)
+
+This product includes slf4j, developed by QOS.ch, in binary form
+(http://www.slf4j.org)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 4e382b7..4a0682c 100644
--- a/README.md
+++ b/README.md
@@ -38,12 +38,16 @@ Currently the wiki contains the most relevant and up-to-date documentation.
Source code is available here: https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git
+Requirements
+------------
+* JDK 6 or higher
+* *nix or macosx (you may build the project and develop S4 applications with microsoft windows though, the only limitation is that the "s4" script has not been ported to windows yet)
+
How to build
------------
This only applies if you checkout from the source repository or if you download a released source package.
-You need a 6+ JDK.
We use gradle http://gradle.org as the build system.
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 006a425..674e4a7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -60,7 +60,6 @@ project.ext["libraries"] = [
slf4j: 'org.slf4j:slf4j-api:1.6.1',
logback_core: 'ch.qos.logback:logback-core:0.9.29',
logback_classic: 'ch.qos.logback:logback-classic:0.9.29',
- ejml: 'com.googlecode.efficient-java-matrix-library:ejml:0.17',
zk: 'org.apache.zookeeper:zookeeper:3.3.1',
jcip: 'net.jcip:jcip-annotations:1.0',
junit: 'junit:junit:4.10',
@@ -210,7 +209,7 @@ task copyRequiredFilesToS4ToolsDistDir (type: Copy) {
-task binDist(dependsOn: ["clean", "s4-tools:clean", "javadocBinRelease", "copyRequiredFilesToS4ToolsDistDir", "s4-tools:cleanDistZip", "s4-tools:distZip"]) {
+task binDist(dependsOn: ["s4-tools:clean", "javadocBinRelease", "copyRequiredFilesToS4ToolsDistDir", "s4-tools:cleanDistZip", "s4-tools:distZip"]) {
doLast {
copy {
from (project.rootDir.path+"/subprojects/s4-tools/build/distributions/")
@@ -220,14 +219,13 @@ task binDist(dependsOn: ["clean", "s4-tools:clean", "javadocBinRelease", "copyRe
}
}
-task srcDist(dependsOn: ["clean", "javadoc"], type: Zip) {
+task srcDist(dependsOn: ["javadoc"], type: Zip) {
doFirst {
platformProjects.collect { project ->
project.tasks.clean.execute()
}
}
classifier='src'
- appendix='zip'
version = project.version
baseName=project.ext.archivesBaseName
destinationDir = project.buildDir
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/s4-example.gradle
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/s4-example.gradle b/subprojects/s4-example/s4-example.gradle
index 8a49de9..f1c663c 100644
--- a/subprojects/s4-example/s4-example.gradle
+++ b/subprojects/s4-example/s4-example.gradle
@@ -16,14 +16,23 @@
* limitations under the License.
*/
-
-description = 'Examples of S4 applications.'
-
-dependencies {
- compile project( ":s4-base" )
- compile project( ":s4-core" )
- compile project( ":s4-comm" )
- compile project( ":s4-edsl" )
- compile libraries.ejml
- compile libraries.junit
-}
+
+
+description = 'Examples of S4 applications.'
+
+
+
+dependencies {
+
+ compile project( ":s4-base" )
+
+ compile project( ":s4-core" )
+
+ compile project( ":s4-comm" )
+
+ compile project( ":s4-edsl" )
+
+ compile libraries.junit
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ClassIDKeyFinder.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ClassIDKeyFinder.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ClassIDKeyFinder.java
deleted file mode 100644
index 0abf109..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ClassIDKeyFinder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.s4.base.KeyFinder;
-
-public class ClassIDKeyFinder implements KeyFinder<ObsEvent> {
-
- @Override
- public List<String> get(ObsEvent event) {
-
- List<String> results = new ArrayList<String>();
-
- /* Retrieve the user ID and add it to the list. */
- results.add(Integer.toString(event.getClassId()));
-
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Controller.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Controller.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Controller.java
deleted file mode 100644
index 06f712b..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Controller.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.s4.core.Receiver;
-import org.apache.s4.core.Sender;
-import org.apache.s4.model.Model;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import ch.qos.logback.classic.Level;
-
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-
-/*
- * Train a classifier, run a test, compute the accuracy of the classifier.
- */
-public class Controller {
-
- private static final Logger logger = LoggerFactory.getLogger(Controller.class);
-
- final private String trainFilename;
- final private String testFilename;
- final private long numTrainVectors;
- final private long numTestVectors;
- final private int numIterations;
- final private int outputInterval;
- final private Model model;
- final private int vectorSize;
- private int numClasses;
- final private Sender sender;
- final private Receiver receiver;
-
- @Inject
- private Controller(@Named("model.train_data") String trainFilename, @Named("model.test_data") String testFilename,
- Model model, @Named("model.vector_size") int vectorSize, @Named("model.num_iterations") int numIterations,
- @Named("model.output_interval_in_seconds") int outputInterval,
- @Named("model.logger.level") String logLevel, Sender sender, Receiver receiver) {
-
- this.trainFilename = trainFilename;
- this.testFilename = testFilename;
- this.numTrainVectors = getNumLines(trainFilename);
- this.numTestVectors = getNumLines(testFilename);
- this.numIterations = numIterations;
- this.vectorSize = vectorSize;
- this.outputInterval = outputInterval;
- this.model = model;
- this.sender = sender;
- this.receiver = receiver;
-
- logger.info("Number of test vectors is " + numTestVectors);
- logger.info("Number of train vectors is " + numTrainVectors);
- ch.qos.logback.classic.Logger root = (ch.qos.logback.classic.Logger) LoggerFactory
- .getLogger(Logger.ROOT_LOGGER_NAME);
- root.setLevel(Level.toLevel(logLevel));
- }
-
- public void start() {
-
- logger.info("Processing file: " + trainFilename);
- try {
-
- /* Get vector size and number of classes from data set. */
- getDataSetInfo(trainFilename);
-
- MyApp app = new MyApp(numClasses, numTrainVectors, model, outputInterval, TimeUnit.SECONDS);
-
- // app.setCommLayer(sender, receiver);
-
- logger.info("Init app.");
- app.initApp();
-
- /* Initialize modelPEs by injecting one dummy events per class. */
- for (int i = 0; i < numClasses; i++) {
- ObsEvent obsEvent = new ObsEvent(-1, new float[vectorSize], -Float.MAX_VALUE, i, -1, true);
- app.injectByKey(obsEvent);
- }
-
- /* Wait until the app is initialized. */
- while (!app.isInited()) {
- Thread.sleep(1);
- }
-
- long start = System.nanoTime();
- for (int i = 0; i < numIterations; i++) {
- logger.info("Starting iteration {}.", i);
- injectData(app, true, trainFilename);
-
- /*
- * Make sure all the data has been processed.
- */
- while (!app.isTrained(i)) {
- Thread.sleep(5);
- }
- }
- long stop = System.nanoTime();
- long trainTime = stop - start;
-
- /* Start testing. */
- logger.info("Start testing.");
- start = System.nanoTime();
- injectData(app, false, testFilename);
- stop = System.nanoTime();
- long testTime = stop - start;
-
- while (!app.isTested(numTestVectors)) {
- Thread.sleep(5);
- }
-
- /* Print final report. */
- logger.info(app.getReport());
-
- /* Print timing info. */
- logger.info("Total training time was {} seconds.", trainTime / 1000000000);
- logger.info("Training time per observation was {} microseconds.", trainTime / numTrainVectors / 1000);
- logger.info("Training time per observation per iteration was {} microseconds.", trainTime / numTrainVectors
- / numIterations / 1000);
- logger.info("Total testing time was {} seconds.", testTime / 1000000000);
- logger.info("Testing time per observation was {} microseconds.", testTime / numTrainVectors / 1000);
-
- /* Done. */
- app.closeApp();
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- logger.error(e.getMessage());
- } catch (IOException e) {
- e.printStackTrace();
- logger.error(e.getMessage());
- } catch (InterruptedException e) {
- e.printStackTrace();
- logger.error(e.getMessage());
- }
- }
-
- private void injectData(MyApp app, boolean isTraining, String filename) throws FileNotFoundException, IOException {
-
- DataFile data = new DataFile(filename);
- int count = 0;
- for (String line : data) {
-
- String[] result = line.split("\\s");
-
- /* Class ID range starts in 1, shift to start in zero. */
- int classID = Integer.parseInt(result[0]) - 1;
-
- float[] vector = new float[vectorSize];
- for (int j = 0; j < vectorSize; j++) {
-
- vector[j] = Float.parseFloat(result[j + 1]);
- }
- ObsEvent obsEvent = new ObsEvent(count++, vector, -Float.MAX_VALUE, classID, -1, isTraining);
- app.injectToAll(obsEvent);
- }
- data.close();
- }
-
- private void getDataSetInfo(String filename) throws FileNotFoundException, IOException {
-
- Map<Integer, Long> countsPerClass = new HashMap<Integer, Long>();
-
- DataFile data = new DataFile(filename);
-
- for (String line : data) {
-
- String[] result = line.split("\\s");
-
- /* Format is: label val1 val2 ... valN */
- if (vectorSize != result.length - 1) {
- throw new IllegalArgumentException("vectorSize: (" + vectorSize
- + ") does not match number of columns in data file (" + (result.length - 1) + ").");
- }
-
- /* Class ID range starts in 1, shift to start in zero. */
- int classID = Integer.parseInt(result[0]) - 1;
-
- /* Count num vectors per class. */
- if (!countsPerClass.containsKey(classID)) {
- countsPerClass.put(classID, 1L);
- } else {
- long count = countsPerClass.get(classID) + 1;
- countsPerClass.put(classID, count);
- }
- }
- data.close();
-
- /* Summary. */
- numClasses = countsPerClass.size();
- logger.info("Number of classes is " + numClasses);
- logger.info("Vector size is " + vectorSize);
-
- for (Map.Entry<Integer, Long> entry : countsPerClass.entrySet()) {
-
- int key = entry.getKey();
- long val = entry.getValue();
-
- logger.info("Num vectors for class ID: " + key + " is " + val);
- }
- }
-
- /*
- * @return Returns the number of lines in a text file.
- */
- private long getNumLines(String filename) {
-
- long count = 0;
- try {
- DataFile data = new DataFile(filename);
-
- for (@SuppressWarnings("unused")
- String line : data) {
- count++;
- }
- data.close();
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- return count;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/DataFile.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/DataFile.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/DataFile.java
deleted file mode 100644
index ec7c33d..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/DataFile.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.*;
-import java.util.zip.GZIPInputStream;
-import java.io.*;
-
-public class DataFile implements Iterable<String> {
- private BufferedReader reader;
-
- public DataFile(String filePath) throws FileNotFoundException, IOException {
-
- InputStream is = this.getClass().getResourceAsStream(filePath);
-
- GZIPInputStream gzip = new GZIPInputStream(is);
-
- this.reader = new BufferedReader(new InputStreamReader(gzip));
- }
-
- public void close() throws IOException {
- reader.close();
- }
-
- public Iterator<String> iterator() {
- return new FileIterator();
- }
-
- private class FileIterator implements Iterator<String> {
- private String currentLine;
-
- public boolean hasNext() {
- try {
- currentLine = reader.readLine();
- } catch (Exception ex) {
- currentLine = null;
- }
-
- return currentLine != null;
- }
-
- public String next() {
- return currentLine;
- }
-
- public void remove() {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/HypIDKeyFinder.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/HypIDKeyFinder.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/HypIDKeyFinder.java
deleted file mode 100644
index 8c5591b..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/HypIDKeyFinder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.s4.base.KeyFinder;
-
-public class HypIDKeyFinder implements KeyFinder<ObsEvent> {
-
- @Override
- public List<String> get(ObsEvent event) {
-
- List<String> results = new ArrayList<String>();
-
- /* Retrieve the user ID and add it to the list. */
- results.add(Integer.toString(event.getHypId()));
-
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Main.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Main.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Main.java
deleted file mode 100644
index 4fa83ba..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Main.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-
-public class Main {
-
- /**
- * @param args
- */
- public static void main(String[] args) {
-
- // Injector injector = Guice.createInjector(new DefaultModule());
- //
- // Controller controller = injector.getInstance(Controller.class);
- // controller.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MaximizerPE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MaximizerPE.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MaximizerPE.java
deleted file mode 100644
index 73b0108..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MaximizerPE.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import org.apache.s4.base.Event;
-import org.apache.s4.core.App;
-import org.apache.s4.core.ProcessingElement;
-import org.apache.s4.core.Stream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-final public class MaximizerPE extends ProcessingElement {
-
- private static final Logger logger = LoggerFactory.getLogger(MaximizerPE.class);
-
- private int numClasses;
- private Stream<ObsEvent> assignmentStream;
- private int numEventsReceived = 0;
- private float maxLogProb = -Float.MAX_VALUE;
- private int hypID;
-
- public MaximizerPE(App app) {
- super(app);
- }
-
- public void setAssignmentStream(Stream<ObsEvent> stream) {
- assignmentStream = stream;
- }
-
- /**
- * @return the numClasses
- */
- public int getNumClasses() {
- return numClasses;
- }
-
- /**
- * @param numClasses
- * the numClasses to set
- */
- public void setNumClasses(int numClasses) {
- this.numClasses = numClasses;
- }
-
- public void onEvent(Event event) {
-
- ObsEvent inEvent = (ObsEvent) event;
- float[] obs = inEvent.getObsVector();
-
- if (inEvent.getProb() > maxLogProb) {
- maxLogProb = inEvent.getProb();
- hypID = inEvent.getHypId();
- }
-
- if (++numEventsReceived == numClasses) {
-
- /* Got all the distances. Send class id with minimum distance. */
- ObsEvent outEvent = new ObsEvent(inEvent.getIndex(), obs, maxLogProb, inEvent.getClassId(), hypID, false);
-
- logger.trace("IN: " + inEvent.toString());
- logger.trace("OUT: " + outEvent.toString());
-
- assignmentStream.put(outEvent);
-
- /* This PE instance is no longer needed. */
- close();
- }
-
- }
-
- @Override
- protected void onRemove() {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- protected void onCreate() {
- // TODO Auto-generated method stub
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MetricsPE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MetricsPE.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MetricsPE.java
deleted file mode 100644
index 394cbee..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MetricsPE.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.Map;
-import java.util.HashMap;
-
-import org.apache.s4.base.Event;
-import org.apache.s4.core.App;
-import org.apache.s4.core.ProcessingElement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-final public class MetricsPE extends ProcessingElement {
-
- private static final Logger logger = LoggerFactory
- .getLogger(MetricsPE.class);
-
- private Map<Integer, HashMap<Integer, MutableInt>> counts;
- private long totalCount = 0;
- private int numClasses;
-
- public MetricsPE(App app) {
- super(app);
- }
-
- public void onEvent(Event event) {
-
- ResultEvent resultEvent = (ResultEvent) event;
- int classID = resultEvent.getClassId();
- int hypID = resultEvent.getHypId();
- totalCount += 1;
-
- /* Increment counter. */
- if (!counts.containsKey(classID)) {
- counts.put(classID, new HashMap<Integer, MutableInt>());
- numClasses++;
- }
- MutableInt value = counts.get(classID).get(hypID);
- if (value == null) {
- value = new MutableInt();
- counts.get(classID).put(hypID, value);
- }
- value.inc();
- }
-
- public void onTrigger(Event event) {
- logger.info(this.toString());
- }
-
- @Override
- protected void onCreate() {
- counts = new HashMap<Integer, HashMap<Integer, MutableInt>>();
- }
-
- @Override
- protected void onRemove() {
- }
-
- /** @return number of data vectors processed. */
- public long getCount() {
- return totalCount;
- }
-
- @Override
- public String toString() {
- StringBuilder report = new StringBuilder();
- report.append("\n\nConfusion Matrix [%]:\n");
- report.append("\n ");
- for (int i = 0; i < numClasses; i++) {
- report.append(String.format("%6d", i));
- }
- report.append("\n ----------------------------------------\n");
- long truePositives = 0;
- for (Map.Entry<Integer, HashMap<Integer, MutableInt>> entry : counts
- .entrySet()) {
-
- int classID = entry.getKey();
- report.append(String.format("%5d:", classID));
- HashMap<Integer, MutableInt> hypCounts = entry.getValue();
-
- long totalCountForClass = getTotalCountForClass(hypCounts);
- float[] sortedCounts = new float[numClasses];
- for (Map.Entry<Integer, MutableInt> hypEntry : hypCounts.entrySet()) {
- int hypID = hypEntry.getKey();
- long count = hypEntry.getValue().get();
-
- /*
- * Because of timing, it is possible to have a hypId that was
- * not counted in numClasses yet. In this case we bail out and
- * without producing a report.
- */
- if (hypID > (numClasses - 1))
- return "Insufficient data.";
-
- sortedCounts[hypID] = (float) count / totalCountForClass * 100f;
- if (classID == hypID)
- truePositives += count;
- }
- for (int i = 0; i < numClasses; i++) {
- report.append(String.format("%6.1f", sortedCounts[i]));
- }
-
- report.append("\n");
- }
- report.append(String.format(
- "\nAccuracy: %6.1f%% - Num Observations: %6d\n",
- (float) truePositives / totalCount * 100f, totalCount));
-
- return report.toString();
- }
-
- private long getTotalCountForClass(HashMap<Integer, MutableInt> counts) {
-
- long count = 0;
- for (Map.Entry<Integer, MutableInt> hypEntry : counts.entrySet()) {
- count += hypEntry.getValue().get();
- }
- return count;
- }
-
- private class MutableInt {
- private int value = 0;
-
- private void inc() {
- ++value;
- }
-
- private int get() {
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ModelPE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ModelPE.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ModelPE.java
deleted file mode 100644
index a252ff4..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ModelPE.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import net.jcip.annotations.ThreadSafe;
-
-import org.apache.s4.base.Event;
-import org.apache.s4.core.App;
-import org.apache.s4.core.ProcessingElement;
-import org.apache.s4.core.Stream;
-import org.apache.s4.model.Model;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-@ThreadSafe
-final public class ModelPE extends ProcessingElement {
-
- private static final Logger logger = LoggerFactory.getLogger(ModelPE.class);
-
- private long numVectors;
- private Model model;
- private Stream<ObsEvent> distanceStream;
- private Stream<ResultEvent> resultStream;
- private int modelId;
- private double logPriorProb;
- private long obsCount = 0;
- private long totalCount = 0;
- private int iteration = 0;
-
- public ModelPE(App app) {
- super(app);
- }
-
- /**
- * @param numVectors the numVectors to set
- */
- public void setNumVectors(long numVectors) {
- this.numVectors = numVectors;
- }
-
- /**
- * @return the number of training vectors.
- */
- public long getNumVectors() {
- return numVectors;
- }
-
- /**
- * @param model
- * the model to set
- */
- public void setModel(Model model) {
- this.model = model;
- }
-
- /**
- * @return the model
- */
- public Model getModel() {
- return model;
- }
-
- /**
- * Set the output streams.
- *
- * @param distanceStream
- * sends an {@link ObsEvent} to the {@link MaximizerPE}.
- * @param resultStream
- * sends a {@link ResultEvent} to the {@link MetricsPE}.
- */
- public void setStream(Stream<ObsEvent> distanceStream,
- Stream<ResultEvent> resultStream) {
-
- /* Init prototype. */
- this.distanceStream = distanceStream;
- this.resultStream = resultStream;
- }
-
- /**
- * @return number of observation vectors used in training iteration.
- */
- public long getObsCount() {
- return obsCount;
- }
-
- /**
- * @return current iteration.
- */
- public int getIteration() {
- return iteration;
- }
-
- private void updateStats(ObsEvent event) {
-
- logger.trace("TRAINING: ModelID: {}, {}", modelId, event.toString());
- model.update(event.getObsVector());
-
- obsCount++;
-
- /* Log info. */
- if (obsCount % 10000 == 0) {
- logger.info("Trained model using {} events with class id {}",
- obsCount, modelId);
- }
- }
-
- private void estimateModel() {
-
- model.estimate();
-
- double prob = (double) obsCount / numVectors;
- logPriorProb = Math.log(prob);
- logger.info("Prior prob: {}", prob);
- logger.info("Update params for model {} is: {}", modelId,
- model.toString());
-
- obsCount = 0;
- totalCount = 0;
- model.clearStatistics();
-
- /* Ready to start next iteration. */
- iteration++;
- }
-
- public void onEvent(Event event) {
-
- ObsEvent inEvent = (ObsEvent) event;
- float[] obs = inEvent.getObsVector();
-
- /* Estimate model parameters using the training data. */
- if (inEvent.isTraining()) {
-
- /*
- * Ignore events with negative index. They are just used to create
- * the PE.
- */
- if (inEvent.getIndex() < 0) {
- return;
- }
-
- if (++totalCount == numVectors) {
-
- /* End of training stream. */
- estimateModel();
-
- /* Could send ack here. */
-
- return;
- }
-
- /* Check if the event belongs to this class. */
- if (inEvent.getClassId() == modelId) {
-
- updateStats(inEvent);
-
- } else {
-
- /* Not needed to compute the mean vector. */
- return;
- }
-
- } else { // scoring
-
- if (inEvent.getHypId() < 0) {
- /* Score observed vector and send it to the maximizer. */
- float dist = (float) (model.logProb(obs) + logPriorProb);
- ObsEvent outEvent = new ObsEvent(inEvent.getIndex(), obs, dist,
- inEvent.getClassId(), modelId, false);
-
- logger.trace(inEvent.getIndex() + " " + inEvent.getClassId()
- + " " + modelId + " " + model.logProb(obs) + " "
- + logPriorProb + " " + dist);
-
- distanceStream.put(outEvent);
-
- } else {
-
- /* Send out result. */
- if (resultStream != null) {
- ResultEvent resultEvent = new ResultEvent(
- inEvent.getIndex(), inEvent.getClassId(),
- inEvent.getHypId());
-
- resultStream.put(resultEvent);
- }
- }
- }
- }
-
- @Override
- protected void onCreate() {
-
- this.modelId = Integer.parseInt(getId());
-
- /*
- * Initialize model. When a new PE instance is created we use the
- * reference to the model in the PE prototype (initial value in variable
- * model) to create a new model for this PE instance (final value in
- * variable model).
- */
- model = model.create();
- }
-
- @Override
- protected void onRemove() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Module.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Module.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Module.java
deleted file mode 100644
index 2564360..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/Module.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.io.InputStream;
-
-import org.apache.commons.configuration.ConfigurationConverter;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.ConfigurationUtils;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.s4.model.Model;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-import com.google.inject.name.Names;
-
-/**
- * Configures the controller.
- *
- *
- */
-public class Module extends AbstractModule {
-
- protected PropertiesConfiguration config = null;
-
- private void loadProperties(Binder binder) {
-
- try {
- InputStream is = this.getClass().getResourceAsStream("/model.properties");
- config = new PropertiesConfiguration();
- config.load(is);
-
- System.out.println(ConfigurationUtils.toString(config));
- // TODO - validate properties.
-
- /* Make all properties injectable. Do we need this? */
- Names.bindProperties(binder, ConfigurationConverter.getProperties(config));
- } catch (ConfigurationException e) {
- binder.addError(e);
- e.printStackTrace();
- }
- }
-
- @Override
- protected void configure() {
- if (config == null)
- loadProperties(binder());
-
- int vectorSize = config.getInt("model.vector_size");
- int numGaussians = config.getInt("model.num_gaussians");
-
- /*
- * Here we bind a specific Model implementation. This one is the GaussianMixtureModel
- */
- bind(Model.class).toInstance(
- new org.apache.s4.model.GaussianMixtureModel(vectorSize, numGaussians,
- org.apache.s4.model.GaussianMixtureModel.TrainMethod.STEP));
-
- /* Alternatively, we could bind to the GaussianModel. */
- // bind(Model.class).toInstance(
- // new org.apache.s4.model.GaussianModel(vectorSize, true));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MyApp.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MyApp.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MyApp.java
deleted file mode 100644
index 4ca32e1..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/MyApp.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.s4.core.App;
-import org.apache.s4.core.ProcessingElement;
-import org.apache.s4.core.Stream;
-import org.apache.s4.model.Model;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MyApp extends App {
-
- private static final Logger logger = LoggerFactory.getLogger(MyApp.class);
-
- final private int numClasses;
- final private long numVectors;
- final private int outputInterval;
- final private TimeUnit timeUnit;
- final private Model model;
- private Stream<ObsEvent> obsStream;
-
- private ModelPE modelPE;
- private MetricsPE metricsPE;
- Stream<ObsEvent> assignmentStream;
- private String report;
-
- MyApp(int numClasses, long numVectors, Model model, int outputInterval, TimeUnit timeUnit) {
- super();
- this.numClasses = numClasses;
- this.numVectors = numVectors;
- this.model = model;
- this.outputInterval = outputInterval;
- this.timeUnit = timeUnit;
- }
-
- public void injectToAll(ObsEvent obsEvent) {
- logger.trace("Inject: " + obsEvent.toString());
- obsStream.put(obsEvent);
- }
-
- public void injectByKey(ObsEvent obsEvent) {
- logger.trace("Inject: " + obsEvent.toString());
- assignmentStream.put(obsEvent);
- }
-
- @Override
- protected void onStart() {
-
- }
-
- /* init() visibility is protected. Normally the Server class only should call init. */
- public void initApp() {
- init();
- }
-
- @Override
- protected void onInit() {
-
- metricsPE = createPE(MetricsPE.class);
-
- Stream<ResultEvent> resultStream = createStream("Result Stream", new ResultKeyFinder(), metricsPE);
-
- modelPE = createPE(ModelPE.class);
- modelPE.setModel(model);
- modelPE.setNumVectors(numVectors);
-
- assignmentStream = createStream("Assignment Stream", new ClassIDKeyFinder(), modelPE);
-
- MaximizerPE maximizerPE = createPE(MaximizerPE.class);
- maximizerPE.setNumClasses(numClasses);
- maximizerPE.setAssignmentStream(assignmentStream);
-
- Stream<ObsEvent> distanceStream = createStream("Distance Stream", new ObsIndexKeyFinder(), maximizerPE);
-
- /*
- * There is a loop in this graph so we need to set the stream at the end. Is there a cleaner way to do this?
- */
- modelPE.setStream(distanceStream, resultStream);
- // modelPE.setOutputIntervalInEvents(10); // output every 10 events
- metricsPE.setTimerInterval(outputInterval, timeUnit); // output every 5
- // seconds
- // obsStream = new Stream<ObsEvent>(this, "Observation Stream", new
- // ClassIDKeyFinder(), modelPE);
- obsStream = createStream("Observation Stream", modelPE);
- }
-
- /** @return true if modelPE is initialized. */
- public boolean isInited() {
- if (modelPE.getRemoteInstances().size() == numClasses)
- return true;
- else
- return false;
- }
-
- /**
- * Check if a training iteration is completed.
- *
- * @param iteration
- * @return true when a training iteration is completed.
- */
- public boolean isTrained(int iteration) {
-
- int sum = 0;
- for (ProcessingElement pe : modelPE.getRemoteInstances().values()) {
- sum += ((ModelPE) pe).getIteration();
- }
- if (sum < iteration * numClasses)
- return false;
- else
- return true;
- }
-
- /**
- * Check if a test is complete.
- *
- * @return true when testing is complete.
- */
- public boolean isTested(long numTestVectors) {
-
- long count = 0;
- MetricsPE mpe = null;
- for (ProcessingElement pe : metricsPE.getRemoteInstances().values()) {
- mpe = (MetricsPE) pe;
- count = mpe.getCount();
- }
- if (count < numTestVectors) {
- return false;
- } else {
- report = mpe.toString();
- return true;
- }
- }
-
- /**
- * @return a metrics report.
- */
- public String getReport() {
-
- return report;
- }
-
- /* close() visibility is protected. Normally the Server class only should call init. */
- public void closeApp() {
- close();
- }
-
- public long getObsCount() {
-
- return modelPE.getObsCount();
- }
-
- @Override
- protected void onClose() {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsEvent.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsEvent.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsEvent.java
deleted file mode 100644
index 4bc816d..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsEvent.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import org.apache.s4.base.Event;
-
-import net.jcip.annotations.Immutable;
-
-@Immutable
-final public class ObsEvent extends Event {
-
- private float[] obsVector;
- private float prob;
- private long index;
- private int classId;
- private int hypId;
- private boolean isTraining;
-
- public ObsEvent() {
-
- }
-
- public ObsEvent(long index, float[] obsVector, float prob, int classId,
- int hypId, boolean isTraining) {
- this.obsVector = obsVector;
- this.prob = prob;
- this.index = index;
- this.classId = classId;
- this.hypId = hypId;
- this.isTraining = isTraining;
- }
-
- /**
- * @return the observed data vector.
- */
- public float[] getObsVector() {
- return obsVector;
- }
-
- /**
- * @return the probability of this event.
- */
- public float getProb() {
- return prob;
- }
-
- /**
- * @return the index of the data vector.
- */
- public long getIndex() {
- return index;
- }
-
- /**
- * @return the true class of the vector.
- */
- public int getClassId() {
- return classId;
- }
-
- /**
- * @return the hypothesized class of the vector. Use -1 when unknown.
- */
- public int getHypId() {
- return hypId;
- }
-
- /**
- * @return true if this is training data.
- */
- public boolean isTraining() {
- return isTraining;
- }
-
- public String toString() {
-
- StringBuilder vector = new StringBuilder();
- for (int i = 0; i < obsVector.length; i++) {
- vector.append(obsVector[i] + " ");
- }
- return "Idx: " + index + ", Class: " + classId + ", Hyp:" + hypId
- + ", Prob: " + prob + ", isTraining: " + isTraining
- + ", Obs: " + vector.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/96eddf87/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsIndexKeyFinder.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsIndexKeyFinder.java b/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsIndexKeyFinder.java
deleted file mode 100644
index 67af387..0000000
--- a/subprojects/s4-example/src/main/java/org/apache/s4/example/model/ObsIndexKeyFinder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.example.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.s4.base.KeyFinder;
-
-public class ObsIndexKeyFinder implements KeyFinder<ObsEvent> {
-
- @Override
- public List<String> get(ObsEvent event) {
-
- List<String> results = new ArrayList<String>();
-
- /* Retrieve the user ID and add it to the list. */
- results.add(Long.toString(event.getIndex()));
-
- return results;
- }
-
-}