You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2016/12/02 07:04:23 UTC
[21/50] [abbrv] incubator-hivemall git commit: add tests
add tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/5088ef36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/5088ef36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/5088ef36
Branch: refs/heads/JIRA-22/pr-385
Commit: 5088ef36367df1cd51ae62f1c044933676975e2e
Parents: a882c5f
Author: amaya <gi...@sapphire.in.net>
Authored: Wed Sep 21 16:22:09 2016 +0900
Committer: amaya <gi...@sapphire.in.net>
Committed: Wed Sep 21 18:00:35 2016 +0900
----------------------------------------------------------------------
.../tools/matrix/TransposeAndDotUDAF.java | 2 +-
.../ftvec/selection/ChiSquareUDFTest.java | 80 ++++++++++++++++++++
.../tools/array/SelectKBeatUDFTest.java | 65 ++++++++++++++++
.../tools/matrix/TransposeAndDotUDAFTest.java | 58 ++++++++++++++
4 files changed, 204 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
index 9d68f93..9df9305 100644
--- a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
+++ b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
@@ -70,7 +70,7 @@ public final class TransposeAndDotUDAF extends AbstractGenericUDAFResolver {
return new TransposeAndDotUDAFEvaluator();
}
- private static final class TransposeAndDotUDAFEvaluator extends GenericUDAFEvaluator {
+ static final class TransposeAndDotUDAFEvaluator extends GenericUDAFEvaluator {
// PARTIAL1 and COMPLETE
private ListObjectInspector matrix0RowOI;
private PrimitiveObjectInspector matrix0ElOI;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java b/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java
new file mode 100644
index 0000000..38f7f57
--- /dev/null
+++ b/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java
@@ -0,0 +1,80 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2016 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.ftvec.selection;
+
+import hivemall.utils.hadoop.WritableUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ChiSquareUDFTest {
+
+ @Test
+ public void test() throws Exception {
+ // this test is based on iris data set
+ final ChiSquareUDF chi2 = new ChiSquareUDF();
+ final List<List<DoubleWritable>> observed = new ArrayList<List<DoubleWritable>>();
+ final List<List<DoubleWritable>> expected = new ArrayList<List<DoubleWritable>>();
+ final GenericUDF.DeferredObject[] dObjs = new GenericUDF.DeferredObject[] {
+ new GenericUDF.DeferredJavaObject(observed),
+ new GenericUDF.DeferredJavaObject(expected)};
+
+ final double[][] matrix0 = new double[][] {
+ {250.29999999999998, 170.90000000000003, 73.2, 12.199999999999996},
+ {296.8, 138.50000000000003, 212.99999999999997, 66.3},
+ {329.3999999999999, 148.7, 277.59999999999997, 101.29999999999998}};
+ final double[][] matrix1 = new double[][] {
+ {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589},
+ {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589},
+ {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589}};
+
+ for (double[] row : matrix0) {
+ observed.add(WritableUtils.toWritableList(row));
+ }
+ for (double[] row : matrix1) {
+ expected.add(WritableUtils.toWritableList(row));
+ }
+
+ chi2.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)),
+ ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector))});
+ final Object[] result = (Object[]) chi2.evaluate(dObjs);
+ final double[] result0 = new double[matrix0[0].length];
+ final double[] result1 = new double[matrix0[0].length];
+ for (int i = 0; i < result0.length; i++) {
+ result0[i] = Double.valueOf(((List) result[0]).get(i).toString());
+ result1[i] = Double.valueOf(((List) result[1]).get(i).toString());
+ }
+
+ final double[] answer0 = new double[] {10.817820878493995, 3.5944990176817315,
+ 116.16984746363957, 67.24482558215503};
+ final double[] answer1 = new double[] {0.004476514990225833, 0.16575416718561453, 0.d,
+ 2.55351295663786e-15};
+
+ Assert.assertArrayEquals(answer0, result0, 0.d);
+ Assert.assertArrayEquals(answer1, result1, 0.d);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java b/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java
new file mode 100644
index 0000000..b86db5c
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java
@@ -0,0 +1,65 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2016 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.tools.array;
+
+import hivemall.utils.hadoop.WritableUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.IntWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+public class SelectKBeatUDFTest {
+
+ @Test
+ public void test() throws Exception {
+ final SelectKBestUDF selectKBest = new SelectKBestUDF();
+ final int k = 2;
+ final double[] data = new double[] {250.29999999999998, 170.90000000000003, 73.2,
+ 12.199999999999996};
+ final double[] importanceList = new double[] {292.1666753739119, 152.70000455081467,
+ 187.93333893418327, 59.93333511948589};
+
+ final GenericUDF.DeferredObject[] dObjs = new GenericUDF.DeferredObject[] {
+ new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(data)),
+ new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(importanceList)),
+ new GenericUDF.DeferredJavaObject(new IntWritable(k))};
+
+ selectKBest.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector});
+ final List resultObj = (List) selectKBest.evaluate(dObjs);
+
+ Assert.assertEquals(resultObj.size(), k);
+
+ final double[] result = new double[k];
+ for (int i = 0; i < k; i++) {
+ result[i] = Double.valueOf(resultObj.get(i).toString());
+ }
+
+ final double[] answer = new double[] {250.29999999999998, 73.2};
+
+ Assert.assertArrayEquals(answer, result, 0.d);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java b/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java
new file mode 100644
index 0000000..93c6ef1
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java
@@ -0,0 +1,58 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2016 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.tools.matrix;
+
+import hivemall.utils.hadoop.WritableUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TransposeAndDotUDAFTest {
+
+ @Test
+ public void test() throws Exception {
+ final TransposeAndDotUDAF tad = new TransposeAndDotUDAF();
+
+ final double[][] matrix0 = new double[][] { {1, -2}, {-1, 3}};
+ final double[][] matrix1 = new double[][] { {1, 2}, {3, 4}};
+
+ final ObjectInspector[] OIs = new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
+ final GenericUDAFEvaluator evaluator = tad.getEvaluator(new SimpleGenericUDAFParameterInfo(
+ OIs, false, false));
+ evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, OIs);
+ TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer agg = (TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer) evaluator.getNewAggregationBuffer();
+ evaluator.reset(agg);
+ for (int i = 0; i < matrix0.length; i++) {
+ evaluator.iterate(agg, new Object[] {WritableUtils.toWritableList(matrix0[i]),
+ WritableUtils.toWritableList(matrix1[i])});
+ }
+
+ final double[][] answer = new double[][] { {-2.0, -2.0}, {7.0, 8.0}};
+
+ for (int i = 0; i < answer.length; i++) {
+ Assert.assertArrayEquals(answer[i], agg.aggMatrix[i], 0.d);
+ }
+ }
+}