You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2015/11/23 03:26:51 UTC
[5/5] hama git commit: HAMA-961: Remove ann package
HAMA-961: Remove ann package
Project: http://git-wip-us.apache.org/repos/asf/hama/repo
Commit: http://git-wip-us.apache.org/repos/asf/hama/commit/3a3ea7a3
Tree: http://git-wip-us.apache.org/repos/asf/hama/tree/3a3ea7a3
Diff: http://git-wip-us.apache.org/repos/asf/hama/diff/3a3ea7a3
Branch: refs/heads/master
Commit: 3a3ea7a37743b5f3759a86460e63ae414b2e9081
Parents: 0225205
Author: Edward J. Yoon <ed...@apache.org>
Authored: Mon Nov 23 11:10:32 2015 +0900
Committer: Edward J. Yoon <ed...@apache.org>
Committed: Mon Nov 23 11:24:49 2015 +0900
----------------------------------------------------------------------
.../apache/hama/commons/math/CrossEntropy.java | 58 --
.../hama/commons/math/FunctionFactory.java | 65 --
.../hama/commons/math/IdentityFunction.java | 36 -
.../org/apache/hama/commons/math/Sigmoid.java | 39 --
.../apache/hama/commons/math/SquaredError.java | 46 --
.../java/org/apache/hama/commons/math/Tanh.java | 36 -
.../hama/commons/math/TestFunctionFactory.java | 82 ---
.../org/apache/hama/bsp/TestPartitioning.java | 2 +-
.../org/apache/hama/examples/ExampleDriver.java | 2 -
.../org/apache/hama/examples/NeuralNetwork.java | 216 ------
.../apache/hama/examples/NeuralNetworkTest.java | 140 ----
.../neuralnets_classification_label.txt | 1 -
.../neuralnets_classification_test.txt | 1 -
.../neuralnets_classification_training.txt | 668 -------------------
.../ml/ann/AbstractLayeredNeuralNetwork.java | 261 --------
.../org/apache/hama/ml/ann/AutoEncoder.java | 197 ------
.../org/apache/hama/ml/ann/NeuralNetwork.java | 271 --------
.../hama/ml/ann/NeuralNetworkTrainer.java | 107 ---
.../hama/ml/ann/SmallLayeredNeuralNetwork.java | 567 ----------------
.../ann/SmallLayeredNeuralNetworkMessage.java | 126 ----
.../ann/SmallLayeredNeuralNetworkTrainer.java | 244 -------
.../apache/hama/ml/perception/MLPMessage.java | 45 --
.../ml/perception/MultiLayerPerceptron.java | 203 ------
.../hama/ml/perception/PerceptronTrainer.java | 96 ---
.../hama/ml/perception/SmallMLPMessage.java | 133 ----
.../hama/ml/perception/SmallMLPTrainer.java | 327 ---------
.../perception/SmallMultiLayerPerceptron.java | 574 ----------------
.../hama/ml/regression/LinearRegression.java | 188 ------
.../hama/ml/regression/LogisticRegression.java | 180 -----
.../org/apache/hama/ml/ann/TestAutoEncoder.java | 195 ------
.../ml/ann/TestSmallLayeredNeuralNetwork.java | 643 ------------------
.../TestSmallLayeredNeuralNetworkMessage.java | 172 -----
.../hama/ml/perception/TestSmallMLPMessage.java | 147 ----
.../TestSmallMultiLayerPerceptron.java | 524 ---------------
.../ml/regression/TestLinearRegression.java | 133 ----
.../ml/regression/TestLogisticRegression.java | 130 ----
36 files changed, 1 insertion(+), 6854 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/CrossEntropy.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/CrossEntropy.java b/commons/src/main/java/org/apache/hama/commons/math/CrossEntropy.java
deleted file mode 100644
index 1378fc4..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/CrossEntropy.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * The cross entropy cost function.
- *
- * <pre>
- * cost(t, y) = - t * log(y) - (1 - t) * log(1 - y),
- * where t denotes the target value, y denotes the estimated value.
- * </pre>
- */
-public class CrossEntropy extends DoubleDoubleFunction {
-
- @Override
- public double apply(double target, double actual) {
- double adjustedTarget = (target == 0 ? 0.000001 : target);
- adjustedTarget = (target == 1.0 ? 0.999999 : target);
- double adjustedActual = (actual == 0 ? 0.000001 : actual);
- adjustedActual = (actual == 1 ? 0.999999 : actual);
- return -adjustedTarget * Math.log(adjustedActual) - (1 - adjustedTarget)
- * Math.log(1 - adjustedActual);
- }
-
- @Override
- public double applyDerivative(double target, double actual) {
- double adjustedTarget = target;
- double adjustedActual = actual;
- if (adjustedActual == 1) {
- adjustedActual = 0.999;
- } else if (actual == 0) {
- adjustedActual = 0.001;
- }
- if (adjustedTarget == 1) {
- adjustedTarget = 0.999;
- } else if (adjustedTarget == 0) {
- adjustedTarget = 0.001;
- }
- return -adjustedTarget / adjustedActual + (1 - adjustedTarget)
- / (1 - adjustedActual);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/FunctionFactory.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/FunctionFactory.java b/commons/src/main/java/org/apache/hama/commons/math/FunctionFactory.java
deleted file mode 100644
index 15c48be..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/FunctionFactory.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * Factory to create the functions.
- *
- */
-public class FunctionFactory {
-
- /**
- * Create a double function with specified name.
- *
- * @param functionName
- * @return an appropriate double function.
- */
- public static DoubleFunction createDoubleFunction(String functionName) {
- if (functionName.equalsIgnoreCase(Sigmoid.class.getSimpleName())) {
- return new Sigmoid();
- } else if (functionName.equalsIgnoreCase(Tanh.class.getSimpleName())) {
- return new Tanh();
- } else if (functionName.equalsIgnoreCase(IdentityFunction.class
- .getSimpleName())) {
- return new IdentityFunction();
- }
-
- throw new IllegalArgumentException(String.format(
- "No double function with name '%s' exists.", functionName));
- }
-
- /**
- * Create a double double function with specified name.
- *
- * @param functionName
- * @return an appropriate double double function.
- */
- public static DoubleDoubleFunction createDoubleDoubleFunction(
- String functionName) {
- if (functionName.equalsIgnoreCase(SquaredError.class.getSimpleName())) {
- return new SquaredError();
- } else if (functionName
- .equalsIgnoreCase(CrossEntropy.class.getSimpleName())) {
- return new CrossEntropy();
- }
-
- throw new IllegalArgumentException(String.format(
- "No double double function with name '%s' exists.", functionName));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/IdentityFunction.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/IdentityFunction.java b/commons/src/main/java/org/apache/hama/commons/math/IdentityFunction.java
deleted file mode 100644
index 6b60aad..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/IdentityFunction.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * The identity function f(x) = x.
- *
- */
-public class IdentityFunction extends DoubleFunction {
-
- @Override
- public double apply(double value) {
- return value;
- }
-
- @Override
- public double applyDerivative(double value) {
- return 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/Sigmoid.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/Sigmoid.java b/commons/src/main/java/org/apache/hama/commons/math/Sigmoid.java
deleted file mode 100644
index eb3e9c6..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/Sigmoid.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * The Sigmoid function
- *
- * <pre>
- * f(x) = 1 / (1 + e^{-x})
- * </pre>
- */
-public class Sigmoid extends DoubleFunction {
-
- @Override
- public double apply(double value) {
- return 1.0 / (1 + Math.exp(-value));
- }
-
- @Override
- public double applyDerivative(double value) {
- return value * (1 - value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/SquaredError.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/SquaredError.java b/commons/src/main/java/org/apache/hama/commons/math/SquaredError.java
deleted file mode 100644
index 42ff81b..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/SquaredError.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * Square error cost function.
- *
- * <pre>
- * cost(t, y) = 0.5 * (t - y) ˆ 2
- * </pre>
- */
-public class SquaredError extends DoubleDoubleFunction {
-
- @Override
- /**
- * {@inheritDoc}
- */
- public double apply(double target, double actual) {
- double diff = target - actual;
- return 0.5 * diff * diff;
- }
-
- @Override
- /**
- * {@inheritDoc}
- */
- public double applyDerivative(double target, double actual) {
- return actual - target;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/main/java/org/apache/hama/commons/math/Tanh.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/hama/commons/math/Tanh.java b/commons/src/main/java/org/apache/hama/commons/math/Tanh.java
deleted file mode 100644
index c1ef6cb..0000000
--- a/commons/src/main/java/org/apache/hama/commons/math/Tanh.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-/**
- * Tanh function.
- *
- */
-public class Tanh extends DoubleFunction {
-
- @Override
- public double apply(double value) {
- return Math.tanh(value);
- }
-
- @Override
- public double applyDerivative(double value) {
- return 1 - value * value;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/commons/src/test/java/org/apache/hama/commons/math/TestFunctionFactory.java
----------------------------------------------------------------------
diff --git a/commons/src/test/java/org/apache/hama/commons/math/TestFunctionFactory.java b/commons/src/test/java/org/apache/hama/commons/math/TestFunctionFactory.java
deleted file mode 100644
index 43a4bcf..0000000
--- a/commons/src/test/java/org/apache/hama/commons/math/TestFunctionFactory.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.commons.math;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Random;
-
-import org.junit.Test;
-
-/**
- * Test case for {@link FunctionFactory}
- *
- */
-public class TestFunctionFactory {
-
- @Test
- public void testCreateDoubleFunction() {
- double input = 0.8;
-
- String sigmoidName = "Sigmoid";
- DoubleFunction sigmoidFunction = FunctionFactory
- .createDoubleFunction(sigmoidName);
- assertEquals(sigmoidName, sigmoidFunction.getFunctionName());
-
- double sigmoidExcepted = 0.68997448;
- assertEquals(sigmoidExcepted, sigmoidFunction.apply(input), 0.000001);
-
-
- String tanhName = "Tanh";
- DoubleFunction tanhFunction = FunctionFactory.createDoubleFunction(tanhName);
- assertEquals(tanhName, tanhFunction.getFunctionName());
-
- double tanhExpected = 0.66403677;
- assertEquals(tanhExpected, tanhFunction.apply(input), 0.00001);
-
-
- String identityFunctionName = "IdentityFunction";
- DoubleFunction identityFunction = FunctionFactory.createDoubleFunction(identityFunctionName);
-
- Random rnd = new Random();
- double identityExpected = rnd.nextDouble();
- assertEquals(identityExpected, identityFunction.apply(identityExpected), 0.000001);
- }
-
- @Test
- public void testCreateDoubleDoubleFunction() {
- double target = 0.5;
- double output = 0.8;
-
- String squaredErrorName = "SquaredError";
- DoubleDoubleFunction squaredErrorFunction = FunctionFactory.createDoubleDoubleFunction(squaredErrorName);
- assertEquals(squaredErrorName, squaredErrorFunction.getFunctionName());
-
- double squaredErrorExpected = 0.045;
-
- assertEquals(squaredErrorExpected, squaredErrorFunction.apply(target, output), 0.000001);
-
- String crossEntropyName = "CrossEntropy";
- DoubleDoubleFunction crossEntropyFunction = FunctionFactory.createDoubleDoubleFunction(crossEntropyName);
- assertEquals(crossEntropyName, crossEntropyFunction.getFunctionName());
-
- double crossEntropyExpected = 0.91629;
- assertEquals(crossEntropyExpected, crossEntropyFunction.apply(target, output), 0.000001);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/core/src/test/java/org/apache/hama/bsp/TestPartitioning.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/hama/bsp/TestPartitioning.java b/core/src/test/java/org/apache/hama/bsp/TestPartitioning.java
index 00c63fd..674c42a 100644
--- a/core/src/test/java/org/apache/hama/bsp/TestPartitioning.java
+++ b/core/src/test/java/org/apache/hama/bsp/TestPartitioning.java
@@ -55,7 +55,7 @@ public class TestPartitioning extends HamaCluster {
configuration.set("bsp.local.dir", "/tmp/hama-test");
configuration.set(Constants.ZOOKEEPER_QUORUM, "localhost");
configuration.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810);
- configuration.set("hama.sync.client.class",
+ configuration.set("hama.sync.peer.class",
org.apache.hama.bsp.sync.ZooKeeperSyncClientImpl.class
.getCanonicalName());
}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/examples/src/main/java/org/apache/hama/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/hama/examples/ExampleDriver.java b/examples/src/main/java/org/apache/hama/examples/ExampleDriver.java
index 08559e6..89d289d 100644
--- a/examples/src/main/java/org/apache/hama/examples/ExampleDriver.java
+++ b/examples/src/main/java/org/apache/hama/examples/ExampleDriver.java
@@ -39,8 +39,6 @@ public class ExampleDriver {
pgd.addClass("semi", SemiClusterJobDriver.class, "Semi Clustering");
pgd.addClass("kmeans", Kmeans.class, "K-Means Clustering");
pgd.addClass("gd", GradientDescentExample.class, "Gradient Descent");
- pgd.addClass("neuralnets", NeuralNetwork.class,
- "Neural Network classification");
pgd.addClass("kcore", KCore.class, "kcore");
pgd.addClass("gen", Generator.class, "Random Data Generator Util");
pgd.driver(args);
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java b/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
deleted file mode 100644
index ef029a6..0000000
--- a/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.examples;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.commons.math.DenseDoubleVector;
-import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
-import org.apache.hama.ml.ann.SmallLayeredNeuralNetwork;
-
-/**
- * The example of using {@link SmallLayeredNeuralNetwork}, including the
- * training phase and labeling phase.
- */
-public class NeuralNetwork {
-
- public static void main(String[] args) throws Exception {
- if (args.length < 3) {
- printUsage();
- return;
- }
- String mode = args[0];
- if (mode.equalsIgnoreCase("label")) {
- if (args.length < 4) {
- printUsage();
- return;
- }
- HamaConfiguration conf = new HamaConfiguration();
-
- String featureDataPath = args[1];
- String resultDataPath = args[2];
- String modelPath = args[3];
-
- SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(modelPath);
-
- // process data in streaming approach
- FileSystem fs = FileSystem.get(new URI(featureDataPath), conf);
- BufferedReader br = new BufferedReader(new InputStreamReader(
- fs.open(new Path(featureDataPath))));
- Path outputPath = new Path(resultDataPath);
- if (fs.exists(outputPath)) {
- fs.delete(outputPath, true);
- }
- BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
- fs.create(outputPath)));
-
- String line = null;
-
- while ((line = br.readLine()) != null) {
- if (line.trim().length() == 0) {
- continue;
- }
- String[] tokens = line.trim().split(",");
- double[] vals = new double[tokens.length];
- for (int i = 0; i < tokens.length; ++i) {
- vals[i] = Double.parseDouble(tokens[i]);
- }
- DoubleVector instance = new DenseDoubleVector(vals);
- DoubleVector result = ann.getOutput(instance);
- double[] arrResult = result.toArray();
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < arrResult.length; ++i) {
- sb.append(arrResult[i]);
- if (i != arrResult.length - 1) {
- sb.append(",");
- } else {
- sb.append("\n");
- }
- }
- bw.write(sb.toString());
- }
-
- br.close();
- bw.close();
- } else if (mode.equals("train")) {
- if (args.length < 5) {
- printUsage();
- return;
- }
-
- String trainingDataPath = args[1];
- String trainedModelPath = args[2];
-
- int featureDimension = Integer.parseInt(args[3]);
- int labelDimension = Integer.parseInt(args[4]);
-
- int iteration = 1000;
- double learningRate = 0.4;
- double momemtumWeight = 0.2;
- double regularizationWeight = 0.01;
-
- // parse parameters
- if (args.length >= 6) {
- try {
- iteration = Integer.parseInt(args[5]);
- System.out.printf("Iteration: %d\n", iteration);
- } catch (NumberFormatException e) {
- System.err
- .println("MAX_ITERATION format invalid. It should be a positive number.");
- return;
- }
- }
- if (args.length >= 7) {
- try {
- learningRate = Double.parseDouble(args[6]);
- System.out.printf("Learning rate: %f\n", learningRate);
- } catch (NumberFormatException e) {
- System.err
- .println("LEARNING_RATE format invalid. It should be a positive double in range (0, 1.0)");
- return;
- }
- }
- if (args.length >= 8) {
- try {
- momemtumWeight = Double.parseDouble(args[7]);
- System.out.printf("Momemtum weight: %f\n", momemtumWeight);
- } catch (NumberFormatException e) {
- System.err
- .println("MOMEMTUM_WEIGHT format invalid. It should be a positive double in range (0, 1.0)");
- return;
- }
- }
- if (args.length >= 9) {
- try {
- regularizationWeight = Double.parseDouble(args[8]);
- System.out
- .printf("Regularization weight: %f\n", regularizationWeight);
- } catch (NumberFormatException e) {
- System.err
- .println("REGULARIZATION_WEIGHT format invalid. It should be a positive double in range (0, 1.0)");
- return;
- }
- }
-
- // train the model
- SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork();
- ann.setLearningRate(learningRate);
- ann.setMomemtumWeight(momemtumWeight);
- ann.setRegularizationWeight(regularizationWeight);
- ann.addLayer(featureDimension, false,
- FunctionFactory.createDoubleFunction("Sigmoid"));
- ann.addLayer(featureDimension, false,
- FunctionFactory.createDoubleFunction("Sigmoid"));
- ann.addLayer(labelDimension, true,
- FunctionFactory.createDoubleFunction("Sigmoid"));
- ann.setCostFunction(FunctionFactory
- .createDoubleDoubleFunction("CrossEntropy"));
- ann.setModelPath(trainedModelPath);
-
- Map<String, String> trainingParameters = new HashMap<String, String>();
- trainingParameters.put("tasks", "5");
- trainingParameters.put("training.max.iterations", "" + iteration);
- trainingParameters.put("training.batch.size", "300");
- trainingParameters.put("convergence.check.interval", "1000");
- ann.train(new Path(trainingDataPath), trainingParameters);
- }
-
- }
-
- private static void printUsage() {
- System.out
- .println("USAGE: <MODE> <INPUT_PATH> <OUTPUT_PATH> <MODEL_PATH>|<FEATURE_DIMENSION> <LABEL_DIMENSION> [<MAX_ITERATION> <LEARNING_RATE> <MOMEMTUM_WEIGHT> <REGULARIZATION_WEIGHT>]");
- System.out
- .println("\tMODE\t- train: train the model with given training data.");
- System.out
- .println("\t\t- label: obtain the result by feeding the features to the neural network.");
- System.out
- .println("\tINPUT_PATH\tin 'train' mode, it is the path of the training data; in 'label' mode, it is the path of the to be evaluated data that lacks the label.");
- System.out
- .println("\tOUTPUT_PATH\tin 'train' mode, it is where the trained model is stored; in 'label' mode, it is where the labeled data is stored.");
- System.out.println("\n\tConditional Parameters:");
- System.out
- .println("\tMODEL_PATH\tonly required in 'label' mode. It specifies where to load the trained neural network model.");
- System.out
- .println("\tMAX_ITERATION\tonly used in 'train' mode. It specifies how many iterations for the neural network to run. Default is 0.01.");
- System.out
- .println("\tLEARNING_RATE\tonly used to 'train' mode. It specifies the degree of aggregation for learning, usually in range (0, 1.0). Default is 0.1.");
- System.out
- .println("\tMOMEMTUM_WEIGHT\tonly used to 'train' mode. It specifies the weight of momemtum. Default is 0.");
- System.out
- .println("\tREGULARIZATION_WEIGHT\tonly required in 'train' model. It specifies the weight of reqularization.");
- System.out.println("\nExample:");
- System.out
- .println("Train a neural network with with feature dimension 8, label dimension 1 and default setting:\n\tneuralnets train hdfs://localhost:30002/training_data hdfs://localhost:30002/model 8 1");
- System.out
- .println("Train a neural network with with feature dimension 8, label dimension 1 and specify learning rate as 0.1, momemtum rate as 0.2, and regularization weight as 0.01:\n\tneuralnets.train hdfs://localhost:30002/training_data hdfs://localhost:30002/model 8 1 0.1 0.2 0.01");
- System.out
- .println("Label the data with trained model:\n\tneuralnets evaluate hdfs://localhost:30002/unlabeled_data hdfs://localhost:30002/result hdfs://localhost:30002/model");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
----------------------------------------------------------------------
diff --git a/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java b/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
deleted file mode 100644
index 6b4798d..0000000
--- a/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.examples;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.commons.io.VectorWritable;
-import org.apache.hama.commons.math.DenseDoubleVector;
-
-/**
- * Test the functionality of NeuralNetwork Example.
- *
- */
-public class NeuralNetworkTest extends TestCase {
- private Configuration conf = new HamaConfiguration();
- private FileSystem fs;
- private String MODEL_PATH = "/tmp/neuralnets.model";
- private String RESULT_PATH = "/tmp/neuralnets.txt";
- private String SEQTRAIN_DATA = "/tmp/test-neuralnets.data";
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- fs = FileSystem.get(conf);
- }
-
- public void testNeuralnetsLabeling() throws IOException {
- this.neuralNetworkTraining();
-
- String dataPath = "src/test/resources/neuralnets_classification_test.txt";
- String mode = "label";
- try {
- NeuralNetwork
- .main(new String[] { mode, dataPath, RESULT_PATH, MODEL_PATH });
-
- // compare results with ground-truth
- BufferedReader groundTruthReader = new BufferedReader(new FileReader(
- "src/test/resources/neuralnets_classification_label.txt"));
- List<Double> groundTruthList = new ArrayList<Double>();
- String line = null;
- while ((line = groundTruthReader.readLine()) != null) {
- groundTruthList.add(Double.parseDouble(line));
- }
- groundTruthReader.close();
-
- BufferedReader resultReader = new BufferedReader(new FileReader(
- RESULT_PATH));
- List<Double> resultList = new ArrayList<Double>();
- while ((line = resultReader.readLine()) != null) {
- resultList.add(Double.parseDouble(line));
- }
- resultReader.close();
- int total = resultList.size();
- double correct = 0;
- for (int i = 0; i < groundTruthList.size(); ++i) {
- double actual = resultList.get(i);
- double expected = groundTruthList.get(i);
- if (actual < 0.5 && expected < 0.5 || actual >= 0.5 && expected >= 0.5) {
- ++correct;
- }
- }
- System.out.printf("Precision: %f\n", correct / total);
-
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- fs.delete(new Path(RESULT_PATH), true);
- fs.delete(new Path(MODEL_PATH), true);
- fs.delete(new Path(SEQTRAIN_DATA), true);
- }
- }
-
- private void neuralNetworkTraining() {
- String mode = "train";
- String strTrainingDataPath = "src/test/resources/neuralnets_classification_training.txt";
- int featureDimension = 8;
- int labelDimension = 1;
-
- Path sequenceTrainingDataPath = new Path(SEQTRAIN_DATA);
- Configuration conf = new Configuration();
- FileSystem fs;
- try {
- fs = FileSystem.get(conf);
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
- sequenceTrainingDataPath, LongWritable.class, VectorWritable.class);
- BufferedReader br = new BufferedReader(
- new FileReader(strTrainingDataPath));
- String line = null;
- // convert the data in sequence file format
- while ((line = br.readLine()) != null) {
- String[] tokens = line.split(",");
- double[] vals = new double[tokens.length];
- for (int i = 0; i < tokens.length; ++i) {
- vals[i] = Double.parseDouble(tokens[i]);
- }
- writer.append(new LongWritable(), new VectorWritable(
- new DenseDoubleVector(vals)));
- }
- writer.close();
- br.close();
- } catch (IOException e1) {
- e1.printStackTrace();
- }
-
- try {
- NeuralNetwork.main(new String[] { mode, SEQTRAIN_DATA,
- MODEL_PATH, "" + featureDimension, "" + labelDimension });
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/examples/src/test/resources/neuralnets_classification_label.txt
----------------------------------------------------------------------
diff --git a/examples/src/test/resources/neuralnets_classification_label.txt b/examples/src/test/resources/neuralnets_classification_label.txt
deleted file mode 100644
index e1b6789..0000000
--- a/examples/src/test/resources/neuralnets_classification_label.txt
+++ /dev/null
@@ -1 +0,0 @@
-1
0
0
0
0
0
0
0
1
1
0
1
0
0
1
0
1
0
0
0
0
0
1
0
1
0
1
0
1
1
0
0
0
0
1
1
0
0
0
1
0
1
1
0
0
1
0
0
1
1
0
0
1
0
0
1
0
0
0
0
0
0
0
1
1
1
0
0
0
0
0
0
1
1
0
0
1
0
0
1
0
1
1
1
0
0
1
1
1
0
1
0
1
0
1
0
0
0
0
1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hama/blob/3a3ea7a3/examples/src/test/resources/neuralnets_classification_test.txt
----------------------------------------------------------------------
diff --git a/examples/src/test/resources/neuralnets_classification_test.txt b/examples/src/test/resources/neuralnets_classification_test.txt
deleted file mode 100644
index b19107d..0000000
--- a/examples/src/test/resources/neuralnets_classification_test.txt
+++ /dev/null
@@ -1 +0,0 @@
-0.588235294,0.557788945,0.573770492,0.272727273,0,0.409836066,0.026900085,0.316666667
0.352941176,0.492462312,0.475409836,0.333333333,0.224586288,0.506706408,0.15029889,0.366666667
0.529411765,0.773869347,0.639344262,0.303030303,0.11820331,0.460506706,0.036720751,0.4
0.352941176,0.829145729,0.557377049,0.262626263,0.19858156,0.500745156,0.236122972,0.466666667
0.058823529,0.497487437,0.475409836,0.101010101,0,0.378539493,0.201964133,0
0.588235294,0.341708543,0.868852459,0.232323232,0.057919622,0.529061103,0.088385995,0.433333333
0.176470588,0.618090452,0.819672131,0.353535354,0.283687943,0.853949329,0.342442357,0.016666667
0.470588235,0.457286432,0.672131148,0,0,0.530551416,0.217335611,0.783333333
0.352941176,0.979899497,0.573770492,0,0,0.460506706,0.106746371,0.166666667
0.529411765,0.783919598,0.704918033,0,0,0.369597615,0.064901793,0.533333333
0,0.467336683,0.491803279,0,0,0.526080477,0.078992314,0.066666667
0.176470588,0.608040201,0.426229508,0,0,0.536512668,0.020922289,0.066666
667
0.117647059,0.507537688,0.475409836,0.171717172,0.313238771,0.360655738,0.228864219,0.033333333
0.117647059,0.281407035,0.459016393,0.282828283,0.053191489,0.360655738,0.108454313,0.016666667
0,0.814070352,0.62295082,0.363636364,0,0.739195231,0.122117848,0.083333333
0,0.477386935,0.524590164,0.393939394,0.124113475,0.664679583,0.122971819,0.016666667
0.235294118,0.628140704,0.655737705,0,0,0.481371088,0.195559351,0.1
0.294117647,0.683417085,0.672131148,0,0,0,0.239965841,0.8
0.117647059,0.648241206,0.606557377,0.262626263,0.242316785,0.494783905,0.219043553,0.066666667
0.176470588,0.653266332,0.524590164,0,0,0.344262295,0.100768574,0.016666667
0.058823529,0.537688442,0.409836066,0.191919192,0,0.421758569,0.043979505,0.133333333
0.058823529,0.703517588,0.606557377,0.262626263,0.212765957,0.359165425,0.320239112,0.033333333
0.058823529,0.72361809,0.672131148,0.464646465,0.212765957,0.687034277,0.109735269,0.416666667
0.470588235,0.537688442,0.655737705,0,0,0.36661699,0.332194705,0.
216666667
0.764705882,0.793969849,0.93442623,0,0,0.630402385,0.076430401,0.383333333
0.117647059,0.608040201,0.573770492,0.323232323,0.112293144,0.58271237,0.34500427,0.033333333
0.411764706,0.648241206,0.557377049,0.494949495,0.147754137,0.573770492,0.154141759,0.366666667
0.117647059,0.452261307,0.491803279,0,0,0.350223547,0.04824936,0.066666667
0.411764706,0.713567839,0.737704918,0.242424242,0.567375887,0.453055142,0.021349274,0.366666667
0.176470588,0.849246231,0.606557377,0.191919192,0.147754137,0.445603577,0.081127242,0.166666667
0,0.497487437,0,0,0,0.372578241,0.074722459,0.016666667
0.235294118,0.638190955,0.721311475,0.111111111,0.18321513,0.514157973,0.222032451,0.116666667
0.235294118,0.592964824,0.573770492,0,0,0.66318927,0.352690009,0.083333333
0.117647059,0.613065327,0.62295082,0.272727273,0.236406619,0.535022355,0.17292912,0.083333333
0.352941176,0.628140704,0.639344262,0.313131313,0,0.411326379,0.20794193,0.466666667
0.058823529,0.844221106,0.721311475,0.292929293,0,
0.521609538,0.353116994,0.516666667
0.117647059,0.648241206,0,0,0,0.573770492,0.096498719,0.333333333
0.235294118,0.552763819,0.62295082,0.202020202,0.11820331,0.423248882,0.017079419,0.1
0.352941176,0.40201005,0.655737705,0.363636364,0,0.59314456,0.042271563,0.116666667
0.588235294,0.577889447,0,0,0,0,0.078138343,0.15
0.117647059,0.638190955,0.37704918,0.212121212,0.395981087,0.51266766,0.041844577,0.016666667
0.529411765,0.824120603,0.639344262,0,0,0.488822653,0.029888984,0.4
0.117647059,0.467336683,0.524590164,0.323232323,0.189125296,0.566318927,0.254483348,0.033333333
0.176470588,0.793969849,0.524590164,0.131313131,0.457446809,0.464977645,0.09265585,0.05
0.294117647,0.633165829,0.639344262,0.272727273,0.026004728,0.441132638,0.154141759,0.316666667
0.588235294,0.648241206,0.508196721,0.363636364,0,0.614008942,0.15499573,0.283333333
0,0.673366834,0.475409836,0.202020202,0.343971631,0.393442623,0.116994022,0
0.176470588,0.512562814,0.606557377,0,0,0.439642325,0.018360376,0.1833333
33
0.411764706,0.939698492,0.409836066,0.333333333,0.463356974,0.505216095,0.319385141,0.216666667
0.176470588,0.869346734,0.639344262,0.393939394,0.218676123,0.503725782,0.38087105,0.166666667
0.588235294,0.472361809,0.590163934,0.181818182,0,0.344262295,0.220751494,0.583333333
0.058823529,0.542713568,0.491803279,0.464646465,0.210401891,0.529061103,0.143894108,0.05
0.294117647,0.487437186,0.62295082,0.272727273,0,0.530551416,0.128095645,0.516666667
0.235294118,0.417085427,0.704918033,0.191919192,0,0.436661699,0.10204953,0.216666667
0.058823529,0.572864322,0.540983607,0.363636364,0.236406619,0.56780924,0.090093937,0
0.058823529,0.748743719,0.557377049,0.292929293,0.150118203,0.436661699,0.115713066,0.35
0.294117647,0.587939698,0.704918033,0.303030303,0.124113475,0.58271237,0.073868488,0.35
0.058823529,0.557788945,0.770491803,0,0,0.488822653,0.079846285,0.4
0.235294118,0.56281407,0.639344262,0.404040404,0,0.587183308,0.067463706,0.283333333
0.058823529,0.582914573,0.639344262,0.29292
9293,0.212765957,0.538002981,0.178479932,0.066666667
0,0.708542714,0.68852459,0.262626263,0,0.482861401,0.151579846,0.016666667
0.117647059,0.879396985,0.721311475,0,0,0.341281669,0.1058924,0.016666667
0.117647059,0.462311558,0.426229508,0,0,0.448584203,0.026900085,0.016666667
0.176470588,0.653266332,0.639344262,0.232323232,0.093380615,0.423248882,0.104611443,0.216666667
0.470588235,0.603015075,0.704918033,0,0,0.423248882,0.077284372,0.016666667
0.117647059,0.874371859,0.721311475,0.373737374,0.141843972,0.66318927,0.242527754,0.05
0.117647059,0.532663317,0.459016393,0.272727273,0.195035461,0.43219076,0.148590948,0.016666667
0.117647059,0.527638191,0.614754098,0,0,0.347242921,0.205807003,0.533333333
0.235294118,0.477386935,0.491803279,0.323232323,0,0.52757079,0.087959009,0.116666667
0,0.633165829,0.704918033,0.272727273,0.141843972,0.408345753,0.186592656,0
0.470588235,0.326633166,0.590163934,0.232323232,0,0.476900149,0.222886422,0.35
0.117647059,0.497487437,0.491803279,0.171717172,
0.189125296,0.545454545,0.160119556,0
0.058823529,0.512562814,0.606557377,0,0,0.588673621,0.091801879,0.35
0.647058824,0.603015075,0.655737705,0.373737374,0.177304965,0.630402385,0.301878736,0.45
0.176470588,0.512562814,0.360655738,0.202020202,0.111111111,0.459016393,0.137489325,0.083333333
0.058823529,0.547738693,0.475409836,0.181818182,0.137115839,0.424739195,0.060204953,0.016666667
0.529411765,0.703517588,0.770491803,0,0,0.48733234,0.280102477,0.4
0.764705882,0.768844221,0.721311475,0.373737374,0.165484634,0.605067064,0.467976089,0.3
0.705882353,0.502512563,0.68852459,0.333333333,0.124113475,0.44709389,0.175064048,0.416666667
0.058823529,0.738693467,0.770491803,0.414141414,0,0.734724292,0.119555935,0.1
0.058823529,0.407035176,0.606557377,0.414141414,0.067375887,0.690014903,0.434671221,0.183333333
0.176470588,0.939698492,0.573770492,0.222222222,0.236406619,0.54247392,0.140905209,0.25
0.352941176,0.814070352,0.508196721,0,0,0.362146051,0.042698548,0.483333333
0.235294118,0.68341708
5,0.573770492,0,0,0.464977645,0.471391973,0.016666667
0.058823529,0.608040201,0.639344262,0.393939394,0.087470449,0.581222057,0.078138343,0.116666667
0.176470588,0.542713568,0.508196721,0.242424242,0,0.387481371,0.061912895,0.066666667
0,0.909547739,0.721311475,0.444444444,0.602836879,0.645305514,0.061485909,0.083333333
0.470588235,0.773869347,0.639344262,0.323232323,0,0.482861401,0.155849701,0.4
0.058823529,0.64321608,0.721311475,0.393939394,0.130023641,0.543964232,0.418018787,0.266666667
0.411764706,0.688442211,0.737704918,0.414141414,0,0.476900149,0.133646456,0.3
0,0.618090452,0.590163934,0,0,0.540983607,0.076857387,0.516666667
0.058823529,0.532663317,0.62295082,0,0,0.558867362,0.050811272,0.083333333
0.352941176,0.954773869,0.754098361,0,0,0.529061103,0.085397096,0.75
0.117647059,0.442211055,0.475409836,0.262626263,0.01891253,0.423248882,0.293766012,0.016666667
0.529411765,0.854271357,0.606557377,0.313131313,0,0.655737705,0.138770282,0.366666667
0.529411765,0.447236181,0.5081967
21,0,0,0.335320417,0.027327071,0.2
0.588235294,0.507537688,0.62295082,0.484848485,0.212765957,0.490312966,0.03970965,0.7
0.117647059,0.613065327,0.573770492,0.272727273,0,0.548435171,0.111870196,0.1
0.294117647,0.608040201,0.590163934,0.232323232,0.132387707,0.390461997,0.071306576,0.15
0.058823529,0.633165829,0.491803279,0,0,0.448584203,0.115713066,0.433333333
\ No newline at end of file