You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/06/20 10:30:16 UTC
[incubator-hivemall] branch libsvm created (now 8dd14c1)
This is an automated email from the ASF dual-hosted git repository.
myui pushed a change to branch libsvm
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git.
at 8dd14c1 Updated function usage page
This branch includes the following new commits:
new 4a8c64c Added to_libsvm_format UDF
new 7d134b1 Added DDLs
new 8dd14c1 Updated function usage page
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-hivemall] 01/03: Added to_libsvm_format UDF
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch libsvm
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
commit 4a8c64c036c77e2edd4ccb72c9f292bc8a8beb0a
Author: Makoto Yui <my...@apache.org>
AuthorDate: Thu Jun 20 19:04:48 2019 +0900
Added to_libsvm_format UDF
---
.../hivemall/ftvec/conv/ToLibSVMFormatUDF.java | 217 +++++++++++++++++++++
.../hivemall/ftvec/conv/ToLibSVMFormatUDFTest.java | 99 ++++++++++
2 files changed, 316 insertions(+)
diff --git a/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java b/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java
new file mode 100644
index 0000000..a1c85d9
--- /dev/null
+++ b/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.ftvec.conv;
+
+import hivemall.UDFWithOptions;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.hashing.MurmurHash3;
+import hivemall.utils.lang.NumberUtils;
+import hivemall.utils.lang.Primitives;
+import hivemall.utils.lang.StringUtils;
+import hivemall.utils.struct.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+@Description(name = "to_libsvm_format",
+ value = "_FUNC_(array<string> feautres [, double/integer target, const string options])"
+ + " - Returns a string representation of libsvm")
+@UDFType(deterministic = true, stateful = false)
+public final class ToLibSVMFormatUDF extends UDFWithOptions {
+
+ private ListObjectInspector _featuresOI;
+ @Nullable
+ private PrimitiveObjectInspector _targetOI = null;
+ private int _numFeatures = MurmurHash3.DEFAULT_NUM_FEATURES;
+ private StringBuilder _buf;
+
+ @Override
+ protected Options getOptions() {
+ Options opts = new Options();
+ opts.addOption("features", "num_features", true,
+ "The number of features [default: 16777217 (2^24)]");
+ return opts;
+ }
+
+ @Override
+ protected CommandLine processOptions(@Nonnull String optionValue) throws UDFArgumentException {
+ CommandLine cl = parseOptions(optionValue);
+ this._numFeatures = Primitives.parseInt(cl.getOptionValue("num_features"), _numFeatures);
+ return cl;
+ }
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ assumeTrue(argOIs.length >= 1 || argOIs.length <= 3,
+ "to_libsvm_format UDF takes 1~3 arguments");
+
+ this._featuresOI = HiveUtils.asListOI(argOIs[0]);
+ if (argOIs.length == 2) {
+ ObjectInspector argOI1 = argOIs[1];
+ if (HiveUtils.isNumberOI(argOI1)) {
+ this._targetOI = HiveUtils.asNumberOI(argOI1);
+ } else if (HiveUtils.isConstString(argOI1)) { // no target
+ String opts = HiveUtils.getConstString(argOI1);
+ processOptions(opts);
+ } else {
+ throw new UDFArgumentException(
+ "Unexpected argument type for 2nd argument: " + argOI1.getTypeName());
+ }
+ } else if (argOIs.length == 3) {
+ this._targetOI = HiveUtils.asNumberOI(argOIs[1]);
+ String opts = HiveUtils.getConstString(argOIs[2]);
+ processOptions(opts);
+ }
+
+ this._buf = new StringBuilder();
+
+ return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ }
+
+ @Nullable
+ @Override
+ public String evaluate(DeferredObject[] args) throws HiveException {
+ final StringBuilder buf = this._buf;
+ StringUtils.clear(buf);
+
+ Object arg0 = args[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+
+ final int featureSize = _featuresOI.getListLength(arg0);
+ List<Pair<Integer, Double>> features = new ArrayList<>(featureSize);
+ for (int i = 0; i < featureSize; i++) {
+ Object e = _featuresOI.getListElement(arg0, i);
+ if (e == null) {
+ continue;
+ }
+ Pair<Integer, Double> fv = parse(e.toString(), _numFeatures);
+ features.add(fv);
+ }
+ Collections.sort(features, comparator);
+
+ if (_targetOI != null) {
+ Object arg1 = args[1].get();
+ if (arg1 == null) {
+ throw new HiveException("Detected NULL for the 2nd argument");
+ }
+ if (HiveUtils.isIntegerOI(_targetOI)) {
+ int label = HiveUtils.getInt(arg1, _targetOI);
+ buf.append(label);
+ } else {
+ double label = HiveUtils.getDouble(arg1, _targetOI);
+ buf.append(label);
+ }
+ buf.append(' ');
+ }
+ for (int i = 0, size = features.size(); i < size; i++) {
+ if (i != 0) {
+ buf.append(' ');
+ }
+ Pair<Integer, Double> fv = features.get(i);
+ buf.append(fv.getKey().intValue());
+ buf.append(':');
+ buf.append(fv.getValue().doubleValue());
+ }
+
+ return buf.toString();
+ }
+
+ @Nonnull
+ public static Pair<Integer, Double> parse(@Nonnull final String fv,
+ @Nonnegative final int numFeatures) throws UDFArgumentException {
+ final int headPos = fv.indexOf(':');
+ if (headPos == -1) {
+ if (NumberUtils.isDigits(fv)) {
+ final int f;
+ try {
+ f = Integer.parseInt(fv);
+ } catch (NumberFormatException e) {
+ throw new UDFArgumentException("Invalid feature value: " + fv);
+ }
+ return new Pair<>(f, 1.d);
+ } else {
+ return new Pair<>(mhash(fv, numFeatures), 1.d);
+ }
+ } else {
+ final int tailPos = fv.lastIndexOf(':');
+ if (headPos != tailPos) {
+ throw new UDFArgumentException("Unsupported feature format: " + fv);
+ }
+ String f = fv.substring(0, headPos);
+ String v = fv.substring(headPos + 1);
+ final double d;
+ try {
+ d = Double.parseDouble(v);
+ } catch (NumberFormatException e) {
+ throw new UDFArgumentException("Invalid feature value: " + fv);
+ }
+ if (NumberUtils.isDigits(f)) {
+ final int i;
+ try {
+ i = Integer.parseInt(f);
+ } catch (NumberFormatException e) {
+ throw new UDFArgumentException("Invalid feature value: " + fv);
+ }
+ return new Pair<>(i, d);
+ } else {
+ return new Pair<>(mhash(f, numFeatures), d);
+ }
+ }
+ }
+
+ private static int mhash(@Nonnull final String word, final int numFeatures) {
+ int r = MurmurHash3.murmurhash3_x86_32(word, 0, word.length(), 0x9747b28c) % numFeatures;
+ if (r < 0) {
+ r += numFeatures;
+ }
+ return r + 1;
+ }
+
+ private static final Comparator<Pair<Integer, Double>> comparator =
+ new Comparator<Pair<Integer, Double>>() {
+ @Override
+ public int compare(Pair<Integer, Double> l, Pair<Integer, Double> r) {
+ return l.getKey().compareTo(r.getKey());
+ }
+ };
+
+ @Override
+ public String getDisplayString(String[] args) {
+ return "to_libsvm_format( " + StringUtils.join(args, ',') + " )";
+ }
+}
diff --git a/core/src/test/java/hivemall/ftvec/conv/ToLibSVMFormatUDFTest.java b/core/src/test/java/hivemall/ftvec/conv/ToLibSVMFormatUDFTest.java
new file mode 100644
index 0000000..6a59058
--- /dev/null
+++ b/core/src/test/java/hivemall/ftvec/conv/ToLibSVMFormatUDFTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.ftvec.conv;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ToLibSVMFormatUDFTest {
+
+ @Test
+ public void testFeatureOnly() throws IOException, HiveException {
+ ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();
+
+ udf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 10")});
+
+ Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(new DeferredObject[] {
+ new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1"))}));
+
+ Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(
+ new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1"))}));
+
+ udf.close();
+ }
+
+ @Test
+ public void testFeatureAndIntLabel() throws IOException, HiveException {
+ ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();
+
+ udf.initialize(
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-features 10")});
+
+ Assert.assertEquals("5 3:2.1 7:3.4",
+ udf.evaluate(new DeferredObject[] {
+ new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1")),
+ new DeferredJavaObject(5)}));
+
+ udf.close();
+ }
+
+ @Test
+ public void testFeatureAndFloatLabel() throws IOException, HiveException {
+ ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();
+
+ udf.initialize(
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-features 10")});
+
+ Assert.assertEquals("5.0 3:2.1 7:3.4",
+ udf.evaluate(
+ new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1")),
+ new DeferredJavaObject(5.f)}));
+
+ udf.close();
+ }
+
+
+
+}
[incubator-hivemall] 03/03: Updated function usage page
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch libsvm
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
commit 8dd14c1e8c818b6a84ddeb8c5dcc3bed7d708de1
Author: Makoto Yui <my...@apache.org>
AuthorDate: Thu Jun 20 19:30:06 2019 +0900
Updated function usage page
---
.../hivemall/ftvec/conv/ToLibSVMFormatUDF.java | 15 ++++++++--
docs/gitbook/misc/funcs.md | 33 ++++++++++++++++++----
2 files changed, 41 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java b/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java
index a1c85d9..723cb0b 100644
--- a/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java
+++ b/core/src/main/java/hivemall/ftvec/conv/ToLibSVMFormatUDF.java
@@ -46,9 +46,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+// @formatter:off
@Description(name = "to_libsvm_format",
value = "_FUNC_(array<string> feautres [, double/integer target, const string options])"
- + " - Returns a string representation of libsvm")
+ + " - Returns a string representation of libsvm",
+ extended = "Usage:\n" +
+ " select to_libsvm_format(array('apple:3.4','orange:2.1'))\n" +
+ " > 6284535:3.4 8104713:2.1\n" +
+ " select to_libsvm_format(array('apple:3.4','orange:2.1'), '-features 10')\n" +
+ " > 3:2.1 7:3.4\n" +
+ " select to_libsvm_format(array('7:3.4','3:2.1'), 5.0)\n" +
+ " > 5.0 3:2.1 7:3.4")
+// @formatter:on
@UDFType(deterministic = true, stateful = false)
public final class ToLibSVMFormatUDF extends UDFWithOptions {
@@ -69,7 +78,9 @@ public final class ToLibSVMFormatUDF extends UDFWithOptions {
@Override
protected CommandLine processOptions(@Nonnull String optionValue) throws UDFArgumentException {
CommandLine cl = parseOptions(optionValue);
- this._numFeatures = Primitives.parseInt(cl.getOptionValue("num_features"), _numFeatures);
+ this._numFeatures = Primitives.parseInt(cl.getOptionValue("num_features"),
+ MurmurHash3.DEFAULT_NUM_FEATURES);
+ assumeTrue(_numFeatures > 0, "num_features must be greater than 0: " + _numFeatures);
return cl;
}
diff --git a/docs/gitbook/misc/funcs.md b/docs/gitbook/misc/funcs.md
index ade9ee3..1b1b280 100644
--- a/docs/gitbook/misc/funcs.md
+++ b/docs/gitbook/misc/funcs.md
@@ -65,13 +65,25 @@ Reference: <a href="https://papers.nips.cc/paper/3848-adaptive-regularization-of
GROUP BY feature
```
-- `train_pa1_regr(array<int|bigint|string> features, float target [, constant string options])` - PA-1 regressor that returns a relation consists of `<int|bigint|string> feature, float weight`. Find PA-1 algorithm detail in http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf
-
-- `train_pa1a_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `<int|bigint|string> feature, float weight`.
+- `train_pa1_regr(array<int|bigint|string> features, float target [, constant string options])` - PA-1 regressor that returns a relation consists of `(int|bigint|string) feature, float weight`.
+ ```sql
+ SELECT
+ feature,
+ avg(weight) as weight
+ FROM
+ (SELECT
+ train_pa1_regr(features,label) as (feature,weight)
+ FROM
+ training_data
+ ) t
+ GROUP BY feature
+ ```
+Reference: <a href="http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf" target="_blank">Koby Crammer et.al., Online Passive-Aggressive Algorithms. Journal of Machine Learning Research, 2006.</a><br/>
+- `train_pa1a_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `(int|bigint|string) feature, float weight`.
-- `train_pa2_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `<int|bigint|string> feature, float weight`.
+- `train_pa2_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `(int|bigint|string) feature, float weight`.
-- `train_pa2a_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `<int|bigint|string> feature, float weight`.
+- `train_pa2a_regr(array<int|bigint|string> features, float target [, constant string options])` - Returns a relation consists of `(int|bigint|string) feature, float weight`.
- `train_regressor(list<string|int|bigint> features, double label [, const string options])` - Returns a relation consists of <string|int|bigint feature, float weight>
```
@@ -261,6 +273,17 @@ Reference: <a href="https://papers.nips.cc/paper/3848-adaptive-regularization-of
- `to_dense_features(array<string> feature_vector, int dimensions)` - Returns a dense feature in array<float>
+- `to_libsvm_format(array<string> feautres [, double/integer target, const string options])` - Returns a string representation of libsvm
+ ```sql
+ Usage:
+ select to_libsvm_format(array('apple:3.4','orange:2.1'))
+ > 6284535:3.4 8104713:2.1
+ select to_libsvm_format(array('apple:3.4','orange:2.1'), '-features 10')
+ > 3:2.1 7:3.4
+ select to_libsvm_format(array('7:3.4','3:2.1'), 5.0)
+ > 5.0 3:2.1 7:3.4
+ ```
+
- `to_sparse_features(array<float> feature_vector)` - Returns a sparse feature in array<string>
## Feature hashing
[incubator-hivemall] 02/03: Added DDLs
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch libsvm
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
commit 7d134b175bb4bdc48e682cebec5975108d3aaabc
Author: Makoto Yui <my...@apache.org>
AuthorDate: Thu Jun 20 19:07:51 2019 +0900
Added DDLs
---
resources/ddl/define-all-as-permanent.hive | 3 +++
resources/ddl/define-all.hive | 4 +++-
resources/ddl/define-all.spark | 3 +++
3 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive
index 0c836f2..ff20c8c 100644
--- a/resources/ddl/define-all-as-permanent.hive
+++ b/resources/ddl/define-all-as-permanent.hive
@@ -288,6 +288,9 @@ CREATE FUNCTION build_bins as 'hivemall.ftvec.binning.BuildBinsUDAF' USING JAR '
DROP FUNCTION IF EXISTS feature_binning;
CREATE FUNCTION feature_binning as 'hivemall.ftvec.binning.FeatureBinningUDF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS to_libsvm_format;
+CREATE FUNCTION to_libsvm_format as 'hivemall.ftvec.conv.ToLibSVMFormatUDF' USING JAR '${hivemall_jar}';
+
--------------------------
-- feature transformers --
--------------------------
diff --git a/resources/ddl/define-all.hive b/resources/ddl/define-all.hive
index e6f7c0b..0495113 100644
--- a/resources/ddl/define-all.hive
+++ b/resources/ddl/define-all.hive
@@ -284,6 +284,9 @@ create temporary function build_bins as 'hivemall.ftvec.binning.BuildBinsUDAF';
drop temporary function if exists feature_binning;
create temporary function feature_binning as 'hivemall.ftvec.binning.FeatureBinningUDF';
+drop temporary function if exists to_libsvm_format;
+create temporary function to_libsvm_format as 'hivemall.ftvec.conv.ToLibSVMFormatUDF';
+
--------------------------
-- feature transformers --
--------------------------
@@ -883,4 +886,3 @@ log(10, n_docs / max2(1,df_t)) + 1.0;
create temporary macro tfidf(tf FLOAT, df_t DOUBLE, n_docs DOUBLE)
tf * (log(10, n_docs / max2(1,df_t)) + 1.0);
-
diff --git a/resources/ddl/define-all.spark b/resources/ddl/define-all.spark
index e3ff216..feadbbf 100644
--- a/resources/ddl/define-all.spark
+++ b/resources/ddl/define-all.spark
@@ -287,6 +287,9 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION build_bins AS 'hivemall.ftvec.binning.
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS feature_binning")
sqlContext.sql("CREATE TEMPORARY FUNCTION feature_binning AS 'hivemall.ftvec.binning.FeatureBinningUDF'")
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS to_libsvm_format")
+sqlContext.sql("CREATE TEMPORARY FUNCTION to_libsvm_format AS 'hivemall.ftvec.conv.ToLibSVMFormatUDF'")
+
/**
* feature transformers
*/