You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2018/06/06 09:09:32 UTC
[1/4] incubator-hivemall git commit: [HIVEMALL-145] Merge Brickhouse
functions
Repository: incubator-hivemall
Updated Branches:
refs/heads/master 7ea6bfd52 -> 494960324
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java b/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
new file mode 100644
index 0000000..0d58b3f
--- /dev/null
+++ b/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.docs;
+
+import static hivemall.docs.utils.MarkdownUtils.asCodeBlock;
+import static hivemall.docs.utils.MarkdownUtils.asInlineCode;
+import static hivemall.docs.utils.MarkdownUtils.asListElement;
+import static hivemall.docs.utils.MarkdownUtils.indent;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.maven.execution.MavenSession;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.reflections.Reflections;
+
+/**
+ * Generate a list of UDFs for documentation.
+ *
+ * @link https://hivemall.incubator.apache.org/userguide/misc/generic_funcs.html
+ * @link https://hivemall.incubator.apache.org/userguide/misc/funcs.html
+ */
+@Mojo(name = "generate-funcs-list", defaultPhase = LifecyclePhase.PROCESS_CLASSES,
+ requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME,
+ configurator = "include-project-dependencies")
+public class FuncsListGeneratorMojo extends AbstractMojo {
+
+ @Parameter(defaultValue = "${basedir}", readonly = true)
+ private File basedir;
+
+ @Parameter(defaultValue = "${session}", readonly = true)
+ private MavenSession session;
+
+ @Parameter(defaultValue = "docs/gitbook/misc/generic_funcs.md")
+ private String pathToGenericFuncs;
+
+ @Parameter(defaultValue = "docs/gitbook/misc/funcs.md")
+ private String pathToFuncs;
+
+ private static final Map<String, List<String>> genericFuncsHeaders = new LinkedHashMap<>();
+ static {
+ genericFuncsHeaders.put("# Array",
+ Arrays.asList("hivemall.tools.array", "hivemall.tools.list"));
+ genericFuncsHeaders.put("# Bitset", Collections.singletonList("hivemall.tools.bits"));
+ genericFuncsHeaders.put("# Compression",
+ Collections.singletonList("hivemall.tools.compress"));
+ genericFuncsHeaders.put("# Datetime", Collections.singletonList("hivemall.tools.datetime"));
+ genericFuncsHeaders.put("# JSON", Collections.singletonList("hivemall.tools.json"));
+ genericFuncsHeaders.put("# Map", Collections.singletonList("hivemall.tools.map"));
+ genericFuncsHeaders.put("# MapReduce", Collections.singletonList("hivemall.tools.mapred"));
+ genericFuncsHeaders.put("# Math", Collections.singletonList("hivemall.tools.math"));
+ genericFuncsHeaders.put("# Matrix", Collections.singletonList("hivemall.tools.matrix"));
+ genericFuncsHeaders.put("# Sanity Checks",
+ Collections.singletonList("hivemall.tools.sanity"));
+ genericFuncsHeaders.put("# Text processing",
+ Collections.singletonList("hivemall.tools.text"));
+ genericFuncsHeaders.put("# Timeseries",
+ Collections.singletonList("hivemall.tools.timeseries"));
+ genericFuncsHeaders.put("# Vector", Collections.singletonList("hivemall.tools.vector"));
+ genericFuncsHeaders.put("# Others", Collections.singletonList("hivemall.tools"));
+ }
+
+ private static final Map<String, List<String>> funcsHeaders = new LinkedHashMap<>();
+ static {
+ funcsHeaders.put("# Regression", Collections.singletonList("hivemall.regression"));
+ funcsHeaders.put("# Classification", Collections.<String>emptyList());
+ funcsHeaders.put("## Binary classification",
+ Collections.singletonList("hivemall.classifier"));
+ funcsHeaders.put("## Multiclass classification",
+ Collections.singletonList("hivemall.classifier.multiclass"));
+ funcsHeaders.put("# Matrix factorization", Collections.singletonList("hivemall.mf"));
+ funcsHeaders.put("# Factorization machines", Collections.singletonList("hivemall.fm"));
+ funcsHeaders.put("# Recommendation", Collections.singletonList("hivemall.recommend"));
+ funcsHeaders.put("# Anomaly detection", Collections.singletonList("hivemall.anomaly"));
+ funcsHeaders.put("# Topic modeling", Collections.singletonList("hivemall.topicmodel"));
+ funcsHeaders.put("# Preprocessing", Collections.singletonList("hivemall.ftvec"));
+ funcsHeaders.put("## Data amplification",
+ Collections.singletonList("hivemall.ftvec.amplify"));
+ funcsHeaders.put("## Feature binning", Collections.singletonList("hivemall.ftvec.binning"));
+ funcsHeaders.put("## Feature format conversion",
+ Collections.singletonList("hivemall.ftvec.conv"));
+ funcsHeaders.put("## Feature hashing", Collections.singletonList("hivemall.ftvec.hashing"));
+ funcsHeaders.put("## Feature paring", Collections.singletonList("hivemall.ftvec.pairing"));
+ funcsHeaders.put("## Ranking", Collections.singletonList("hivemall.ftvec.ranking"));
+ funcsHeaders.put("## Feature scaling", Collections.singletonList("hivemall.ftvec.scaling"));
+ funcsHeaders.put("## Feature selection",
+ Collections.singletonList("hivemall.ftvec.selection"));
+ funcsHeaders.put("## Feature transformation and vectorization",
+ Collections.singletonList("hivemall.ftvec.trans"));
+ funcsHeaders.put("# Geospatial functions",
+ Collections.singletonList("hivemall.geospatial"));
+ funcsHeaders.put("# Distance measures", Collections.singletonList("hivemall.knn.distance"));
+ funcsHeaders.put("# Locality-sensitive hashing",
+ Collections.singletonList("hivemall.knn.lsh"));
+ funcsHeaders.put("# Similarity measures",
+ Collections.singletonList("hivemall.knn.similarity"));
+ funcsHeaders.put("# Evaluation", Collections.singletonList("hivemall.evaluation"));
+ funcsHeaders.put("# Sketching",
+ Arrays.asList("hivemall.sketch.hll", "hivemall.sketch.bloom"));
+ funcsHeaders.put("# Ensemble learning", Collections.singletonList("hivemall.ensemble"));
+ funcsHeaders.put("## Bagging", Collections.singletonList("hivemall.ensemble.bagging"));
+ funcsHeaders.put("# Decision trees and RandomForest", Arrays.asList(
+ "hivemall.smile.classification", "hivemall.smile.regression", "hivemall.smile.tools"));
+ funcsHeaders.put("# XGBoost", Arrays.asList("hivemall.xgboost.classification",
+ "hivemall.xgboost.regression", "hivemall.xgboost.tools"));
+ funcsHeaders.put("# Others",
+ Arrays.asList("hivemall", "hivemall.dataset", "hivemall.ftvec.text"));
+ }
+
+ @Override
+ public void execute() throws MojoExecutionException {
+ if (!isReactorRootProject()) {
+ // output only once across the projects
+ return;
+ }
+
+ generate(new File(basedir, pathToGenericFuncs),
+ "This page describes a list of useful Hivemall generic functions. See also a [list of machine-learning-related functions](./funcs.md).",
+ genericFuncsHeaders);
+ generate(new File(basedir, pathToFuncs),
+ "This page describes a list of Hivemall functions. See also a [list of generic Hivemall functions](./generic_funcs.md) for more general-purpose functions such as array and map UDFs.",
+ funcsHeaders);
+ }
+
+ private boolean isReactorRootProject() {
+ return session.getExecutionRootDirectory().equalsIgnoreCase(basedir.toString());
+ }
+
+ private void generate(@Nonnull File outputFile, @Nonnull String preface,
+ @Nonnull Map<String, List<String>> headers) throws MojoExecutionException {
+ Reflections reflections = new Reflections("hivemall");
+ Set<Class<?>> annotatedClasses = reflections.getTypesAnnotatedWith(Description.class);
+
+ StringBuilder sb = new StringBuilder();
+ Map<String, Set<String>> packages = new HashMap<>();
+
+ Pattern func = Pattern.compile("_FUNC_(\\(.*?\\))(.*)", Pattern.DOTALL);
+
+ for (Class<?> annotatedClass : annotatedClasses) {
+ Deprecated deprecated = annotatedClass.getAnnotation(Deprecated.class);
+ if (deprecated != null) {
+ continue;
+ }
+
+ Description description = annotatedClass.getAnnotation(Description.class);
+
+ String value = description.value().replaceAll("\n", " ");
+ Matcher matcher = func.matcher(value);
+ if (matcher.find()) {
+ value = asInlineCode(description.name() + matcher.group(1))
+ + escapeHtml(matcher.group(2));
+ }
+ sb.append(asListElement(value));
+
+ StringBuilder sbExtended = new StringBuilder();
+ if (!description.extended().isEmpty()) {
+ sbExtended.append(description.extended());
+ sb.append("\n");
+ }
+
+ String extended = sbExtended.toString();
+ if (extended.isEmpty()) {
+ sb.append("\n");
+ } else {
+ if (extended.toLowerCase().contains("select")) { // extended description contains SQL statements
+ sb.append(indent(asCodeBlock(extended, "sql")));
+ } else {
+ sb.append(indent(asCodeBlock(extended)));
+ }
+ }
+
+ String packageName = annotatedClass.getPackage().getName();
+ if (!packages.containsKey(packageName)) {
+ Set<String> set = new TreeSet<>();
+ packages.put(packageName, set);
+ }
+ Set<String> List = packages.get(packageName);
+ List.add(sb.toString());
+
+ sb.setLength(0);
+ }
+
+ try (PrintWriter writer = new PrintWriter(outputFile)) {
+ // license header
+ writer.println("<!--");
+ try {
+ File licenseFile = new File(basedir, "resources/license-header.txt");
+ FileReader fileReader = new FileReader(licenseFile);
+
+ try (BufferedReader bufferedReader = new BufferedReader(fileReader)) {
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ writer.println(indent(line));
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to read license file");
+ }
+ writer.println("-->\n");
+
+ writer.println(preface);
+
+ writer.println("\n<!-- toc -->\n");
+
+ for (Map.Entry<String, List<String>> e : headers.entrySet()) {
+ writer.println(e.getKey() + "\n");
+ List<String> packageNames = e.getValue();
+ for (String packageName : packageNames) {
+ if (!packages.containsKey(packageName)) {
+ writer.close();
+ throw new MojoExecutionException(
+ "Failed to find package in the classpath: " + packageName);
+ }
+ for (String desc : packages.get(packageName)) {
+ writer.println(desc);
+ }
+ }
+ }
+
+ writer.flush();
+ } catch (FileNotFoundException e) {
+ throw new MojoExecutionException("Output file is not found");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/tools/hivemall-docs/src/main/java/hivemall/docs/IncludeProjectDependenciesComponentConfigurator.java
----------------------------------------------------------------------
diff --git a/tools/hivemall-docs/src/main/java/hivemall/docs/IncludeProjectDependenciesComponentConfigurator.java b/tools/hivemall-docs/src/main/java/hivemall/docs/IncludeProjectDependenciesComponentConfigurator.java
new file mode 100644
index 0000000..29774f0
--- /dev/null
+++ b/tools/hivemall-docs/src/main/java/hivemall/docs/IncludeProjectDependenciesComponentConfigurator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.docs;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
+import org.codehaus.classworlds.ClassRealm;
+import org.codehaus.plexus.component.annotations.Component;
+import org.codehaus.plexus.component.configurator.AbstractComponentConfigurator;
+import org.codehaus.plexus.component.configurator.ComponentConfigurationException;
+import org.codehaus.plexus.component.configurator.ConfigurationListener;
+import org.codehaus.plexus.component.configurator.converters.composite.ObjectWithFieldsConverter;
+import org.codehaus.plexus.component.configurator.converters.special.ClassRealmConverter;
+import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException;
+import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluator;
+import org.codehaus.plexus.configuration.PlexusConfiguration;
+
+/**
+ * A custom ComponentConfigurator which adds the project's runtime classpath elements to the MOJO
+ * classloader.
+ */
+@SuppressWarnings("deprecation")
+@Component(role = org.codehaus.plexus.component.configurator.ComponentConfigurator.class,
+ hint = "include-project-dependencies")
+public class IncludeProjectDependenciesComponentConfigurator extends AbstractComponentConfigurator {
+
+ public void configureComponent(final Object component, final PlexusConfiguration configuration,
+ final ExpressionEvaluator expressionEvaluator, final ClassRealm containerRealm,
+ final ConfigurationListener listener) throws ComponentConfigurationException {
+ addProjectDependenciesToClassRealm(expressionEvaluator, containerRealm);
+
+ converterLookup.registerConverter(new ClassRealmConverter(containerRealm));
+
+ ObjectWithFieldsConverter converter = new ObjectWithFieldsConverter();
+
+ converter.processConfiguration(converterLookup, component, containerRealm.getClassLoader(),
+ configuration, expressionEvaluator, listener);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void addProjectDependenciesToClassRealm(
+ final ExpressionEvaluator expressionEvaluator, final ClassRealm containerRealm)
+ throws ComponentConfigurationException {
+ final List<String> runtimeClasspathElements;
+ try {
+ // noinspection unchecked
+ runtimeClasspathElements = (List<String>) expressionEvaluator.evaluate(
+ "${project.runtimeClasspathElements}");
+ } catch (ExpressionEvaluationException e) {
+ throw new ComponentConfigurationException(
+ "There was a problem evaluating: ${project.runtimeClasspathElements}", e);
+ }
+
+ // Add the project dependencies to the ClassRealm
+ final URL[] urls = buildURLs(runtimeClasspathElements);
+ for (URL url : urls) {
+ containerRealm.addConstituent(url);
+ }
+ }
+
+ @Nonnull
+ private static URL[] buildURLs(@Nonnull final List<String> runtimeClasspathElements)
+ throws ComponentConfigurationException {
+ // Add the projects classes and dependencies
+ final List<URL> urls = new ArrayList<>(runtimeClasspathElements.size());
+ for (String element : runtimeClasspathElements) {
+ try {
+ URL url = new File(element).toURI().toURL();
+ urls.add(url);
+ } catch (MalformedURLException e) {
+ throw new ComponentConfigurationException(
+ "Unable to access project dependency: " + element, e);
+ }
+ }
+ return urls.toArray(new URL[urls.size()]);
+ }
+
+}
[3/4] incubator-hivemall git commit: [HIVEMALL-145] Merge Brickhouse
functions
Posted by my...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/json/ToJsonUDF.java b/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
index 70c62b9..95d1af9 100644
--- a/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
+++ b/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
@@ -37,8 +37,82 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Text;
+// @formatter:off
@Description(name = "to_json",
- value = "_FUNC_(ANY object [, const array<string>|const string columnNames]) - Returns Json string")
+ value = "_FUNC_(ANY object [, const array<string>|const string columnNames]) - Returns Json string",
+ extended = "SELECT \n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31)\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31),\n" +
+ " array('Name', 'age')\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31),\n" +
+ " array('name', 'age')\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31),\n" +
+ " array('age')\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"Name\", \"John\", \"age\", 31),\n" +
+ " array()\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " null,\n" +
+ " array()\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " struct(\"123\", \"456\", 789, array(314,007)),\n" +
+ " array('ti','si','i','bi')\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " struct(\"123\", \"456\", 789, array(314,007)),\n" +
+ " 'ti,si,i,bi'\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " struct(\"123\", \"456\", 789, array(314,007))\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\")\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " array('city')\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " ARRAY(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"osaka\")\n" +
+ " )\n" +
+ " ),\n" +
+ " to_json(\n" +
+ " ARRAY(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"osaka\")\n" +
+ " ),\n" +
+ " array('city')\n" +
+ " );\n" +
+ "```\n\n" +
+ "```\n" +
+ " {\"name\":\"John\",\"age\":31}\n"
+ + " {\"name\":\"John\",\"age\":31}\n"
+ + " {\"Name\":\"John\",\"age\":31}\n"
+ + " {\"name\":\"John\",\"age\":31}\n"
+ + " {\"age\":31}\n"
+ + " {}\n"
+ + " NULL\n"
+ + " {\"ti\":\"123\",\"si\":\"456\",\"i\":789,\"bi\":[314,7]}\n"
+ + " {\"ti\":\"123\",\"si\":\"456\",\"i\":789,\"bi\":[314,7]}\n"
+ + " {\"col1\":\"123\",\"col2\":\"456\",\"col3\":789,\"col4\":[314,7]}\n"
+ + " {\"country\":\"japan\",\"city\":\"tokyo\"}\n"
+ + " {\"city\":\"tokyo\"}\n"
+ + " [{\"country\":\"japan\",\"city\":\"tokyo\"},{\"country\":\"japan\",\"city\":\"osaka\"}]\n"
+ + " [{\"country\":\"japan\",\"city\":\"tokyo\"},{\"country\":\"japan\",\"city\":\"osaka\"}]")
+// @formatter:on
@UDFType(deterministic = true, stateful = false)
public final class ToJsonUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/list/UDAFToOrderedList.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/list/UDAFToOrderedList.java b/core/src/main/java/hivemall/tools/list/UDAFToOrderedList.java
index 5ef6ddb..83adb0f 100644
--- a/core/src/main/java/hivemall/tools/list/UDAFToOrderedList.java
+++ b/core/src/main/java/hivemall/tools/list/UDAFToOrderedList.java
@@ -66,15 +66,16 @@ import org.apache.hadoop.io.IntWritable;
/**
* Return list of values sorted by value itself or specific key.
*/
+//@formatter:off
@Description(name = "to_ordered_list",
value = "_FUNC_(PRIMITIVE value [, PRIMITIVE key, const string options])"
+ " - Return list of values sorted by value itself or specific key",
- extended = "with t as (\n" + " select 5 as key, 'apple' as value\n" + " union all\n"
- + " select 3 as key, 'banana' as value\n" + " union all\n"
- + " select 4 as key, 'candy' as value\n" + " union all\n"
- + " select 2 as key, 'donut' as value\n" + " union all\n"
- + " select 3 as key, 'egg' as value\n" + ")\n"
- + "select -- expected output\n"
+ extended = "WITH t as (\n" + " SELECT 5 as key, 'apple' as value\n" + " UNION ALL\n"
+ + " SELECT 3 as key, 'banana' as value\n" + " UNION ALL\n"
+ + " SELECT 4 as key, 'candy' as value\n" + " UNION ALL\n"
+ + " SELECT 2 as key, 'donut' as value\n" + " UNION ALL\n"
+ + " SELECT 3 as key, 'egg' as value\n" + ")\n"
+ + "SELECT -- expected output\n"
+ " to_ordered_list(value, key, '-reverse'), -- [apple, candy, (banana, egg | egg, banana), donut] (reverse order)\n"
+ " to_ordered_list(value, key, '-k 2'), -- [apple, candy] (top-k)\n"
+ " to_ordered_list(value, key, '-k 100'), -- [apple, candy, (banana, egg | egg, banana), dunut]\n"
@@ -86,7 +87,8 @@ import org.apache.hadoop.io.IntWritable;
+ " to_ordered_list(value, '-k 2'), -- [egg, donut] (alphabetically)\n"
+ " to_ordered_list(key, '-k -2 -reverse'), -- [5, 4] (top-2 keys)\n"
+ " to_ordered_list(key) -- [2, 3, 3, 4, 5] (natural ordered keys)\n"
- + "from\n" + " t")
+ + "FROM\n" + " t")
+//@formatter:on
public final class UDAFToOrderedList extends AbstractGenericUDAFResolver {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/map/MapExcludeKeysUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/MapExcludeKeysUDF.java b/core/src/main/java/hivemall/tools/map/MapExcludeKeysUDF.java
new file mode 100644
index 0000000..6a46210
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/map/MapExcludeKeysUDF.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import hivemall.utils.hadoop.HiveUtils;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+
+@Description(name = "map_exclude_keys",
+ value = "_FUNC_(Map<K,V> map, array<K> filteringKeys)"
+ + " - Returns the filtered entries of a map not having specified keys",
+ extended = "SELECT map_exclude_keys(map(1,'one',2,'two',3,'three'),array(2,3));\n"
+ + "{1:\"one\"}")
+@UDFType(deterministic = true, stateful = false)
+public final class MapExcludeKeysUDF extends GenericUDF {
+
+ private MapObjectInspector mapOI;
+ private ListObjectInspector listOI;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 2) {
+ throw new UDFArgumentLengthException(
+ "Expected two arguments for map_filter_keys: " + argOIs.length);
+ }
+
+ this.mapOI = HiveUtils.asMapOI(argOIs[0]);
+ this.listOI = HiveUtils.asListOI(argOIs[1]);
+
+ ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
+ ObjectInspector filterKeyOI = listOI.getListElementObjectInspector();
+
+ if (!ObjectInspectorUtils.compareTypes(mapKeyOI, filterKeyOI)) {
+ throw new UDFArgumentException("Element types does not match: mapKey "
+ + mapKeyOI.getTypeName() + ", filterKey" + filterKeyOI.getTypeName());
+ }
+
+ return ObjectInspectorUtils.getStandardObjectInspector(mapOI,
+ ObjectInspectorCopyOption.WRITABLE);
+ }
+
+ @Override
+ public Map<?, ?> evaluate(DeferredObject[] arguments) throws HiveException {
+ Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+ final Map<?, ?> map = (Map<?, ?>) ObjectInspectorUtils.copyToStandardObject(arg0, mapOI,
+ ObjectInspectorCopyOption.WRITABLE);
+
+ Object arg1 = arguments[1].get();
+ if (arg1 == null) {
+ return map;
+ }
+
+ final List<?> filterKeys = (List<?>) ObjectInspectorUtils.copyToStandardObject(arg1, listOI,
+ ObjectInspectorCopyOption.WRITABLE);
+ for (Object k : filterKeys) {
+ map.remove(k);
+ }
+
+ return map;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "map_exclude_keys(" + StringUtils.join(children, ',') + ")";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/map/MapIncludeKeysUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/MapIncludeKeysUDF.java b/core/src/main/java/hivemall/tools/map/MapIncludeKeysUDF.java
new file mode 100644
index 0000000..902569f
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/map/MapIncludeKeysUDF.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import hivemall.utils.hadoop.HiveUtils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+
+@Description(name = "map_include_keys",
+ value = "_FUNC_(Map<K,V> map, array<K> filteringKeys)"
+ + " - Returns the filtered entries of a map having specified keys",
+ extended = "SELECT map_include_keys(map(1,'one',2,'two',3,'three'),array(2,3));\n"
+ + "{2:\"two\",3:\"three\"}")
+@UDFType(deterministic = true, stateful = false)
+public final class MapIncludeKeysUDF extends GenericUDF {
+
+ private MapObjectInspector mapOI;
+ private ListObjectInspector listOI;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 2) {
+ throw new UDFArgumentLengthException(
+ "Expected two arguments for map_filter_keys: " + argOIs.length);
+ }
+
+ this.mapOI = HiveUtils.asMapOI(argOIs[0]);
+ this.listOI = HiveUtils.asListOI(argOIs[1]);
+
+ ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
+ ObjectInspector filterKeyOI = listOI.getListElementObjectInspector();
+ if (!ObjectInspectorUtils.compareTypes(mapKeyOI, filterKeyOI)) {
+ throw new UDFArgumentException("Element types does not match: mapKey "
+ + mapKeyOI.getTypeName() + ", filterKey" + filterKeyOI.getTypeName());
+ }
+
+ return ObjectInspectorUtils.getStandardObjectInspector(mapOI,
+ ObjectInspectorCopyOption.WRITABLE);
+ }
+
+ @Override
+ public Map<?, ?> evaluate(DeferredObject[] arguments) throws HiveException {
+ Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+ final Map<?, ?> map = (Map<?, ?>) ObjectInspectorUtils.copyToStandardObject(arg0, mapOI,
+ ObjectInspectorCopyOption.WRITABLE);
+
+ Object arg1 = arguments[1].get();
+ if (arg1 == null) {
+ return null;
+ }
+ final List<?> filterKeys = (List<?>) ObjectInspectorUtils.copyToStandardObject(arg1, listOI,
+ ObjectInspectorCopyOption.WRITABLE);
+
+ final Map<Object, Object> result = new HashMap<>();
+ for (Object k : filterKeys) {
+ Object v = map.get(k);
+ if (v != null) {
+ result.put(k, v);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "map_include_keys(" + StringUtils.join(children, ',') + ")";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/map/MapIndexUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/MapIndexUDF.java b/core/src/main/java/hivemall/tools/map/MapIndexUDF.java
new file mode 100644
index 0000000..73ffa36
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/map/MapIndexUDF.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+//@formatter:off
+@Description(name = "map_index",
+ value = "_FUNC_(a, n) - Returns the n-th element of the given array",
+ extended = "WITH tmp as (\n" +
+ " SELECT \"one\" as key\n" +
+ " UNION ALL\n" +
+ " SELECT \"two\" as key\n" +
+ ")\n" +
+ "SELECT map_index(map(\"one\",1,\"two\",2),key)\n" +
+ "FROM tmp;\n\n" +
+ "1\n" +
+ "2")
+//@formatter:on
+@UDFType(deterministic = true, stateful = false)
+public final class MapIndexUDF extends GenericUDF {
+
+ private transient MapObjectInspector mapOI;
+ private transient Converter converter;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 2) {
+ throw new UDFArgumentLengthException("The function INDEX accepts exactly 2 arguments.");
+ }
+
+ if (arguments[0] instanceof MapObjectInspector) {
+ this.mapOI = (MapObjectInspector) arguments[0];
+ } else {
+ throw new UDFArgumentTypeException(0, "\"map\" is expected at function INDEX, but \""
+ + arguments[0].getTypeName() + "\" is found");
+ }
+
+ // index has to be a primitive
+ if (!(arguments[1] instanceof PrimitiveObjectInspector)) {
+ throw new UDFArgumentTypeException(1,
+ "Primitive Type is expected but " + arguments[1].getTypeName() + "\" is found");
+ }
+
+ PrimitiveObjectInspector inputOI = (PrimitiveObjectInspector) arguments[1];
+ ObjectInspector indexOI =
+ ObjectInspectorConverters.getConvertedOI(inputOI, mapOI.getMapKeyObjectInspector());
+ this.converter = ObjectInspectorConverters.getConverter(inputOI, indexOI);
+
+ return mapOI.getMapValueObjectInspector();
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ assert (arguments.length == 2);
+ Object index = arguments[1].get();
+
+ Object indexObject = converter.convert(index);
+ if (indexObject == null) {
+ return null;
+ }
+
+ Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+
+ return mapOI.getMapValueElement(arg0, indexObject);
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ assert (children.length == 2);
+ return children[0] + "[" + children[1] + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/map/MapKeyValuesUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/MapKeyValuesUDF.java b/core/src/main/java/hivemall/tools/map/MapKeyValuesUDF.java
new file mode 100644
index 0000000..3992f9e
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/map/MapKeyValuesUDF.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+
+@Description(name = "map_key_values",
+ value = "_FUNC_(map) - " + "Returns a array of key-value pairs.",
+ extended = "SELECT map_key_values(map(\"one\",1,\"two\",2));\n\n"
+ + "[{\"key\":\"one\",\"value\":1},{\"key\":\"two\",\"value\":2}]")
+@UDFType(deterministic = true, stateful = false)
+public final class MapKeyValuesUDF extends GenericUDF {
+
+ private final ArrayList<Object[]> retArray = new ArrayList<Object[]>();
+
+ private MapObjectInspector mapOI;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 1) {
+ throw new UDFArgumentLengthException(
+ "The function MAP_KEYS only accepts one argument.");
+ } else if (!(arguments[0] instanceof MapObjectInspector)) {
+ throw new UDFArgumentTypeException(0,
+ "\"" + Category.MAP.toString().toLowerCase()
+ + "\" is expected at function MAP_KEYS, " + "but \""
+ + arguments[0].getTypeName() + "\" is found");
+ }
+
+ this.mapOI = (MapObjectInspector) arguments[0];
+
+ List<String> structFieldNames = new ArrayList<String>();
+ List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
+ structFieldNames.add("key");
+ structFieldObjectInspectors.add(mapOI.getMapKeyObjectInspector());
+ structFieldNames.add("value");
+ structFieldObjectInspectors.add(mapOI.getMapValueObjectInspector());
+
+ return ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors));
+ }
+
+ @Override
+ @Nullable
+ public List<Object[]> evaluate(DeferredObject[] arguments) throws HiveException {
+ Object mapObj = arguments[0].get();
+ if (mapObj == null) {
+ return null;
+ }
+ retArray.clear();
+ final Map<?, ?> map = mapOI.getMap(mapObj);
+ for (Map.Entry<?, ?> e : map.entrySet()) {
+ retArray.add(new Object[] {e.getKey(), e.getValue()});
+ }
+ return retArray;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "map_key_values(" + StringUtils.join(children, ',') + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/map/MergeMapsUDAF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/map/MergeMapsUDAF.java b/core/src/main/java/hivemall/tools/map/MergeMapsUDAF.java
new file mode 100644
index 0000000..e4a2516
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/map/MergeMapsUDAF.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.Preconditions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+//@formatter:off
+@Description(name = "merge_maps",
+ value = "_FUNC_(x) - Returns a map which contains the union of an aggregation of maps."
+ + " Note that an existing value of a key can be replaced with the other duplicate key entry.",
+ extended = "SELECT \n" +
+ " merge_maps(m) \n" +
+ "FROM (\n" +
+ " SELECT map('A',10,'B',20,'C',30) \n" +
+ " UNION ALL \n" +
+ " SELECT map('A',10,'B',20,'C',30)\n" +
+ ") t")
+//@formatter:on
+public final class MergeMapsUDAF extends AbstractGenericUDAFResolver {
+
+ @Override
+ public MergeMapsEvaluator getEvaluator(TypeInfo[] types) throws SemanticException {
+ if (types.length != 1) {
+ throw new UDFArgumentTypeException(types.length - 1,
+ "One argument is expected but got " + types.length);
+ }
+ TypeInfo paramType = types[0];
+ if (paramType.getCategory() != Category.MAP) {
+ throw new UDFArgumentTypeException(0, "Only maps supported for now ");
+ }
+ return new MergeMapsEvaluator();
+ }
+
+ public static final class MergeMapsEvaluator extends GenericUDAFEvaluator {
+
+ private transient MapObjectInspector inputMapOI, mergeMapOI;
+ private transient ObjectInspector inputKeyOI, inputValOI;
+
+ @AggregationType(estimable = false)
+ static final class MapAggBuffer extends AbstractAggregationBuffer {
+ @Nonnull
+ final Map<Object, Object> collectMap = new HashMap<Object, Object>();
+ }
+
+ public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
+ Preconditions.checkArgument(parameters.length == 1);
+ super.init(mode, parameters);
+
+ // initialize input
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
+ this.inputMapOI = HiveUtils.asMapOI(parameters[0]);
+ this.inputKeyOI = inputMapOI.getMapKeyObjectInspector();
+ this.inputValOI = inputMapOI.getMapValueObjectInspector();
+ } else {// from partial aggregation
+ this.mergeMapOI = HiveUtils.asMapOI(parameters[0]);
+ this.inputKeyOI = mergeMapOI.getMapKeyObjectInspector();
+ this.inputValOI = mergeMapOI.getMapValueObjectInspector();
+ }
+
+ return ObjectInspectorFactory.getStandardMapObjectInspector(
+ ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
+ ObjectInspectorUtils.getStandardObjectInspector(inputValOI));
+ }
+
+ @Override
+ public MapAggBuffer getNewAggregationBuffer() throws HiveException {
+ MapAggBuffer buff = new MapAggBuffer();
+ reset(buff);
+ return buff;
+ }
+
+ @Override
+ public void reset(@SuppressWarnings("deprecation") AggregationBuffer buff)
+ throws HiveException {
+ MapAggBuffer aggrBuf = (MapAggBuffer) buff;
+ aggrBuf.collectMap.clear();
+ }
+
+ @Override
+ public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
+ Object[] parameters) throws HiveException {
+ Preconditions.checkArgument(parameters.length == 1);
+
+ Object param0 = parameters[0];
+ if (param0 == null) {
+ return;
+ }
+
+ Map<?, ?> m = inputMapOI.getMap(param0);
+ MapAggBuffer myagg = (MapAggBuffer) agg;
+ putIntoSet(m, myagg.collectMap, inputMapOI);
+ }
+
+ @Override
+ public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
+ throws HiveException {
+ if (partial == null) {
+ return;
+ }
+
+ MapAggBuffer myagg = (MapAggBuffer) agg;
+ Map<?, ?> m = mergeMapOI.getMap(partial);
+ putIntoSet(m, myagg.collectMap, mergeMapOI);
+ }
+
+ private static void putIntoSet(@Nonnull final Map<?, ?> m,
+ @Nonnull final Map<Object, Object> dst, @Nonnull final MapObjectInspector mapOI) {
+ final ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
+ final ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
+
+ for (Map.Entry<?, ?> e : m.entrySet()) {
+ Object k = e.getKey();
+ Object v = e.getValue();
+ Object keyCopy = ObjectInspectorUtils.copyToStandardObject(k, keyOI);
+ Object valCopy = ObjectInspectorUtils.copyToStandardObject(v, valueOI);
+ dst.put(keyCopy, valCopy);
+ }
+ }
+
+ @Override
+ @Nonnull
+ public Map<Object, Object> terminatePartial(
+ @SuppressWarnings("deprecation") AggregationBuffer agg) throws HiveException {
+ MapAggBuffer myagg = (MapAggBuffer) agg;
+ return myagg.collectMap;
+ }
+
+ @Override
+ public Object terminate(@SuppressWarnings("deprecation") AggregationBuffer agg)
+ throws HiveException {
+ MapAggBuffer myagg = (MapAggBuffer) agg;
+ return myagg.collectMap;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java b/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
index 3c92e30..95c97dc 100644
--- a/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
+++ b/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.io.LongWritable;
-@Description(name = "rownum", value = "_FUNC_() - Returns a generated row number in long",
- extended = "returns sprintf(`%d%04d`,sequence,taskId) as long")
+@Description(name = "rownum", value = "_FUNC_() - Returns a generated row number `sprintf(`%d%04d`,sequence,taskId)` in long",
+ extended = "SELECT rownum() as rownum, xxx from ...")
@UDFType(deterministic = false, stateful = true)
public final class RowNumberUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/sanity/AssertUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/sanity/AssertUDF.java b/core/src/main/java/hivemall/tools/sanity/AssertUDF.java
index d34cd20..c74aeb8 100644
--- a/core/src/main/java/hivemall/tools/sanity/AssertUDF.java
+++ b/core/src/main/java/hivemall/tools/sanity/AssertUDF.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
@Description(name = "assert",
value = "_FUNC_(boolean condition) or _FUNC_(boolean condition, string errMsg)"
- + "- Throws HiveException if condition is not met")
-@UDFType(deterministic = true, stateful = false)
+ + "- Throws HiveException if condition is not met",
+ extended = "SELECT count(1) FROM stock_price WHERE assert(price > 0.0);\n"
+ + "SELECT count(1) FROM stock_price WHERE assert(price > 0.0, 'price MUST be more than 0.0')")
+@UDFType(deterministic = false, stateful = false)
public final class AssertUDF extends UDF {
public boolean evaluate(boolean condition) throws HiveException {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java b/core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java
index 194085c..fb6b8eb 100644
--- a/core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java
+++ b/core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java
@@ -18,21 +18,48 @@
*/
package hivemall.tools.sanity;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-@Description(name = "raise_error", value = "_FUNC_() or _FUNC_(string msg) - Throws an error")
-@UDFType(deterministic = true, stateful = false)
-public final class RaiseErrorUDF extends UDF {
+@Description(name = "raise_error", value = "_FUNC_() or _FUNC_(string msg) - Throws an error",
+ extended = "SELECT product_id, price, raise_error('Found an invalid record') FROM xxx WHERE price < 0.0")
+@UDFType(deterministic = false, stateful = false)
+public class RaiseErrorUDF extends GenericUDF {
- public boolean evaluate() throws HiveException {
- throw new HiveException();
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 0 && argOIs.length != 1) {
+ throw new UDFArgumentLengthException(
+ "Expected one or two arguments for raise_error UDF: " + argOIs.length);
+ }
+
+ return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ if (arguments.length == 1) {
+ Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ throw new HiveException();
+ }
+ String msg = arg0.toString();
+ throw new HiveException(msg);
+ } else {
+ throw new HiveException();
+ }
}
- public boolean evaluate(String errorMessage) throws HiveException {
- throw new HiveException(errorMessage);
+ @Override
+ public String getDisplayString(String[] children) {
+ return "raise_error(" + StringUtils.join(children, ',') + ')';
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/Base91UDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/Base91UDF.java b/core/src/main/java/hivemall/tools/text/Base91UDF.java
index 6f52599..73f365b 100644
--- a/core/src/main/java/hivemall/tools/text/Base91UDF.java
+++ b/core/src/main/java/hivemall/tools/text/Base91UDF.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.io.Text;
@Description(name = "base91",
value = "_FUNC_(BINARY bin) - Convert the argument from binary to a BASE91 string",
- extended = "select base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));\n" + "> AA+=kaIM|WTt!+wbGAA")
+ extended = "SELECT base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));\n" + " AA+=kaIM|WTt!+wbGAA")
@UDFType(deterministic = true, stateful = false)
public final class Base91UDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/NormalizeUnicodeUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/NormalizeUnicodeUDF.java b/core/src/main/java/hivemall/tools/text/NormalizeUnicodeUDF.java
index aefb4e2..9a7b547 100644
--- a/core/src/main/java/hivemall/tools/text/NormalizeUnicodeUDF.java
+++ b/core/src/main/java/hivemall/tools/text/NormalizeUnicodeUDF.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
@Description(name = "normalize_unicode",
value = "_FUNC_(string str [, string form]) - Transforms `str` with the specified normalization form. "
+ "The `form` takes one of NFC (default), NFD, NFKC, or NFKD",
- extended = "select normalize_unicode('ﾊﾝｶｸｶﾅ','NFKC');\n" + "> ハンカクカナ\n" + "\n"
- + "select normalize_unicode('㈱㌧㌦Ⅲ','NFKC');\n" + "> (株)トンドルIII")
+ extended = "SELECT normalize_unicode('ﾊﾝｶｸｶﾅ','NFKC');\n" + " ハンカクカナ\n" + "\n"
+ + "SELECT normalize_unicode('㈱㌧㌦Ⅲ','NFKC');\n" + " (株)トンドルIII")
@UDFType(deterministic = true, stateful = false)
public final class NormalizeUnicodeUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/SingularizeUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/SingularizeUDF.java b/core/src/main/java/hivemall/tools/text/SingularizeUDF.java
index 73d2d63..3b16828 100644
--- a/core/src/main/java/hivemall/tools/text/SingularizeUDF.java
+++ b/core/src/main/java/hivemall/tools/text/SingularizeUDF.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
// https://github.com/clips/pattern/blob/3eef00481a4555331cf9a099308910d977f6fc22/pattern/text/en/inflect.py#L445-L623
@Description(name = "singularize",
value = "_FUNC_(string word) - Returns singular form of a given English word",
- extended = "select singularize(lower(\"Apples\"));\n" + "\n" + "> \"apple\"")
+ extended = "SELECT singularize(lower(\"Apples\"));\n" + "\n" + " \"apple\"")
@UDFType(deterministic = true, stateful = false)
public final class SingularizeUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/SplitWordsUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/SplitWordsUDF.java b/core/src/main/java/hivemall/tools/text/SplitWordsUDF.java
index 0b10c2f..31d155d 100644
--- a/core/src/main/java/hivemall/tools/text/SplitWordsUDF.java
+++ b/core/src/main/java/hivemall/tools/text/SplitWordsUDF.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.io.Text;
@Description(name = "split_words",
- value = "_FUNC_(string query [, string regex]) - Returns an array<text> containing split strings")
+ value = "_FUNC_(string query [, string regex]) - Returns an array<text> containing splitted strings")
@UDFType(deterministic = true, stateful = false)
public final class SplitWordsUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/Unbase91UDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/Unbase91UDF.java b/core/src/main/java/hivemall/tools/text/Unbase91UDF.java
index a96b3bf..9c277ce 100644
--- a/core/src/main/java/hivemall/tools/text/Unbase91UDF.java
+++ b/core/src/main/java/hivemall/tools/text/Unbase91UDF.java
@@ -39,8 +39,8 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
@Description(name = "unbase91", value = "_FUNC_(string) - Convert a BASE91 string to a binary",
- extended = "select inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));\n"
- + "> aaaaaaaaaaaaaaaabbbbccc")
+ extended = "SELECT inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));\n"
+ + " aaaaaaaaaaaaaaaabbbbccc")
@UDFType(deterministic = true, stateful = false)
public final class Unbase91UDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/text/WordNgramsUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/text/WordNgramsUDF.java b/core/src/main/java/hivemall/tools/text/WordNgramsUDF.java
index fa8308b..9b3658f 100644
--- a/core/src/main/java/hivemall/tools/text/WordNgramsUDF.java
+++ b/core/src/main/java/hivemall/tools/text/WordNgramsUDF.java
@@ -37,8 +37,8 @@ import java.util.List;
@Description(name = "word_ngrams",
value = "_FUNC_(array<string> words, int minSize, int maxSize])"
+ " - Returns list of n-grams for given words, where `minSize <= n <= maxSize`",
- extended = "select word_ngrams(tokenize('Machine learning is fun!', true), 1, 2);\n" + "\n"
- + "> [\"machine\",\"machine learning\",\"learning\",\"learning is\",\"is\",\"is fun\",\"fun\"]")
+ extended = "SELECT word_ngrams(tokenize('Machine learning is fun!', true), 1, 2);\n" + "\n"
+ + " [\"machine\",\"machine learning\",\"learning\",\"learning is\",\"is\",\"is fun\",\"fun\"]")
@UDFType(deterministic = true, stateful = false)
public final class WordNgramsUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/timeseries/MovingAverageUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/timeseries/MovingAverageUDTF.java b/core/src/main/java/hivemall/tools/timeseries/MovingAverageUDTF.java
new file mode 100644
index 0000000..b5267a1
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/timeseries/MovingAverageUDTF.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.timeseries;
+
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.stats.MovingAverage;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Writable;
+
+// @formatter:off
+@Description(name = "moving_avg",
+ value = "_FUNC_(NUMBER value, const int windowSize)"
+ + " - Returns moving average of a time series using a given window",
+ extended = "SELECT moving_avg(x, 3) FROM (SELECT explode(array(1.0,2.0,3.0,4.0,5.0,6.0,7.0)) as x) series;\n" +
+ " 1.0\n" +
+ " 1.5\n" +
+ " 2.0\n" +
+ " 3.0\n" +
+ " 4.0\n" +
+ " 5.0\n" +
+ " 6.0")
+// @formatter:on
+@UDFType(deterministic = false, stateful = true)
+public final class MovingAverageUDTF extends GenericUDTF {
+
+ private PrimitiveObjectInspector valueOI;
+
+ private MovingAverage movingAvg;
+
+ private Writable[] forwardObjs;
+ private DoubleWritable result;
+
+ @Override
+ public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 2) {
+ throw new UDFArgumentException(
+ "Two argument is expected for moving_avg(NUMBER value, const int windowSize): "
+ + argOIs.length);
+ }
+ this.valueOI = HiveUtils.asNumberOI(argOIs[0]);
+
+ int windowSize = HiveUtils.getConstInt(argOIs[1]);
+ this.movingAvg = new MovingAverage(windowSize);
+
+ this.result = new DoubleWritable();
+ this.forwardObjs = new Writable[] {result};
+
+ List<String> fieldNames = Arrays.asList("avg");
+ List<ObjectInspector> fieldOIs = Arrays.<ObjectInspector>asList(
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ public void process(Object[] args) throws HiveException {
+ double x = HiveUtils.getDouble(args[0], valueOI);
+
+ double avg = movingAvg.add(x);
+ result.set(avg);
+
+ forward(forwardObjs);
+ }
+
+ @Override
+ public void close() throws HiveException {}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/vector/VectorAddUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/vector/VectorAddUDF.java b/core/src/main/java/hivemall/tools/vector/VectorAddUDF.java
index 8442ae3..98fab99 100644
--- a/core/src/main/java/hivemall/tools/vector/VectorAddUDF.java
+++ b/core/src/main/java/hivemall/tools/vector/VectorAddUDF.java
@@ -42,7 +42,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
@Description(name = "vector_add",
- value = "_FUNC_(array<NUMBER> x, array<NUMBER> y) - Perform vector ADD operation.")
+ value = "_FUNC_(array<NUMBER> x, array<NUMBER> y) - Perform vector ADD operation.",
+ extended = "SELECT vector_add(array(1.0,2.0,3.0), array(2, 3, 4));\n" + "[3.0,5.0,7.0]")
@UDFType(deterministic = true, stateful = false)
public final class VectorAddUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/vector/VectorDotUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/vector/VectorDotUDF.java b/core/src/main/java/hivemall/tools/vector/VectorDotUDF.java
index 2aa3c03..b43b562 100644
--- a/core/src/main/java/hivemall/tools/vector/VectorDotUDF.java
+++ b/core/src/main/java/hivemall/tools/vector/VectorDotUDF.java
@@ -43,7 +43,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
@Description(name = "vector_dot",
value = "_FUNC_(array<NUMBER> x, array<NUMBER> y) - Performs vector dot product.",
- extended = "_FUNC_(array<NUMBER> x, NUMBER y) - Performs vector multiplication")
+ extended = "SELECT vector_dot(array(1.0,2.0,3.0),array(2.0,3.0,4.0));\n20\n\n"
+ + "SELECT vector_dot(array(1.0,2.0,3.0),2);\n[2.0,4.0,6.0]")
@UDFType(deterministic = true, stateful = false)
public final class VectorDotUDF extends GenericUDF {
@@ -65,19 +66,19 @@ public final class VectorDotUDF extends GenericUDF {
ObjectInspector argOI1 = argOIs[1];
if (HiveUtils.isNumberListOI(argOI1)) {
this.evaluator = new Dot2DVectors(xListOI, HiveUtils.asListOI(argOI1));
+ return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
} else if (HiveUtils.isNumberOI(argOI1)) {
this.evaluator = new Multiply2D1D(xListOI, argOI1);
+ return ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
} else {
throw new UDFArgumentException(
"Expected array<number> or number for the send argument: " + argOI1.getTypeName());
}
-
- return ObjectInspectorFactory.getStandardListObjectInspector(
- PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
}
@Override
- public List<Double> evaluate(DeferredObject[] args) throws HiveException {
+ public Object evaluate(DeferredObject[] args) throws HiveException {
final Object arg0 = args[0].get();
final Object arg1 = args[1].get();
if (arg0 == null || arg1 == null) {
@@ -90,7 +91,7 @@ public final class VectorDotUDF extends GenericUDF {
interface Evaluator extends Serializable {
@Nonnull
- List<Double> dot(@Nonnull Object x, @Nonnull Object y) throws HiveException;
+ Object dot(@Nonnull Object x, @Nonnull Object y) throws HiveException;
}
@@ -144,7 +145,7 @@ public final class VectorDotUDF extends GenericUDF {
}
@Override
- public List<Double> dot(@Nonnull Object x, @Nonnull Object y) throws HiveException {
+ public Double dot(@Nonnull Object x, @Nonnull Object y) throws HiveException {
final int xLen = xListOI.getListLength(x);
final int yLen = yListOI.getListLength(y);
if (xLen != yLen) {
@@ -152,7 +153,7 @@ public final class VectorDotUDF extends GenericUDF {
+ ", y=" + yListOI.getList(y));
}
- final Double[] arr = new Double[xLen];
+ double result = 0.d;
for (int i = 0; i < xLen; i++) {
Object xi = xListOI.getListElement(x, i);
Object yi = yListOI.getListElement(y, i);
@@ -162,10 +163,10 @@ public final class VectorDotUDF extends GenericUDF {
double xd = PrimitiveObjectInspectorUtils.getDouble(xi, xElemOI);
double yd = PrimitiveObjectInspectorUtils.getDouble(yi, yElemOI);
double v = xd * yd;
- arr[i] = Double.valueOf(v);
+ result += v;
}
- return Arrays.asList(arr);
+ return Double.valueOf(result);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
index f3fe703..12b0e97 100644
--- a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
+++ b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java
@@ -1211,7 +1211,7 @@ public final class HiveUtils {
@Nonnull
public static ObjectInspector getObjectInspector(@Nonnull final String typeString,
- boolean preferWritable) {
+ final boolean preferWritable) {
TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeString);
if (preferWritable) {
return TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/utils/math/MatrixUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/utils/math/MatrixUtils.java b/core/src/main/java/hivemall/utils/math/MatrixUtils.java
index 6c08a61..38329c1 100644
--- a/core/src/main/java/hivemall/utils/math/MatrixUtils.java
+++ b/core/src/main/java/hivemall/utils/math/MatrixUtils.java
@@ -239,7 +239,6 @@ public final class MatrixUtils {
Preconditions.checkArgument(dim >= 1, "Invalid dimension: " + dim);
Preconditions.checkArgument(c.length >= dim,
"|c| must be greater than " + dim + ": " + c.length);
-
/*
* Toeplitz matrix (symmetric, invertible, k*dimensions by k*dimensions)
*
@@ -511,8 +510,12 @@ public final class MatrixUtils {
}
/**
- * Find the first singular vector/value of a matrix A based on the Power method.
+ * Find the first singular vector/value of a matrix A based on the Power method. <<<<<<< HEAD
*
+ * =======
+ *
+ * >>>>>>> Applied spotless-maven-plugin formatter
+ *
* @see http
* ://www.cs.yale.edu/homes/el327/datamining2013aFiles/07_singular_value_decomposition.pdf
* @param A target matrix
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java b/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
index 16196eb..5b7aa8f 100644
--- a/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
+++ b/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
@@ -85,6 +85,7 @@ public class FieldAwareFactorizationMachineUDTFTest {
@Test
public void testSample() throws IOException, HiveException {
+ System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2");
run("[Sample.ffm] default option",
"https://github.com/myui/ml_dataset/raw/master/ffm/sample.ffm.gz",
"-classification -factors 2 -iters 10 -feature_hashing 20 -seed 43", 0.01f);
@@ -92,6 +93,7 @@ public class FieldAwareFactorizationMachineUDTFTest {
// TODO @Test
public void testSampleEnableNorm() throws IOException, HiveException {
+ System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2");
run("[Sample.ffm] default option",
"https://github.com/myui/ml_dataset/raw/master/ffm/sample.ffm.gz",
"-classification -factors 2 -iters 10 -feature_hashing 20 -seed 43 -enable_norm",
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java b/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
index d91eb66..06277c2 100644
--- a/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
+++ b/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
@@ -21,6 +21,9 @@ package hivemall.ftvec.hashing;
import hivemall.TestUtils;
import hivemall.utils.hashing.MurmurHash3;
+import java.io.IOException;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -29,9 +32,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.Arrays;
-
public class FeatureHashingUDFTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/smile/classification/DecisionTreeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/smile/classification/DecisionTreeTest.java b/core/src/test/java/hivemall/smile/classification/DecisionTreeTest.java
index 018de82..b789e71 100644
--- a/core/src/test/java/hivemall/smile/classification/DecisionTreeTest.java
+++ b/core/src/test/java/hivemall/smile/classification/DecisionTreeTest.java
@@ -19,6 +19,7 @@
package hivemall.smile.classification;
import static org.junit.Assert.assertEquals;
+
import hivemall.math.matrix.Matrix;
import hivemall.math.matrix.builders.CSRMatrixBuilder;
import hivemall.math.matrix.dense.RowMajorDenseMatrix2d;
@@ -29,6 +30,10 @@ import hivemall.smile.tools.TreeExportUDF.Evaluator;
import hivemall.smile.tools.TreeExportUDF.OutputType;
import hivemall.smile.utils.SmileExtUtils;
import hivemall.utils.codec.Base91;
+import smile.data.AttributeDataset;
+import smile.data.parser.ArffParser;
+import smile.math.Math;
+import smile.validation.LOOCV;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -43,11 +48,6 @@ import org.apache.hadoop.io.Text;
import org.junit.Assert;
import org.junit.Test;
-import smile.data.AttributeDataset;
-import smile.data.parser.ArffParser;
-import smile.math.Math;
-import smile.validation.LOOCV;
-
public class DecisionTreeTest {
private static final boolean DEBUG = false;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/smile/regression/RegressionTreeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/smile/regression/RegressionTreeTest.java b/core/src/test/java/hivemall/smile/regression/RegressionTreeTest.java
index a4a7f05..9d24b54 100644
--- a/core/src/test/java/hivemall/smile/regression/RegressionTreeTest.java
+++ b/core/src/test/java/hivemall/smile/regression/RegressionTreeTest.java
@@ -27,6 +27,8 @@ import hivemall.smile.data.Attribute.NumericAttribute;
import hivemall.smile.tools.TreeExportUDF.Evaluator;
import hivemall.smile.tools.TreeExportUDF.OutputType;
import hivemall.utils.codec.Base91;
+import smile.math.Math;
+import smile.validation.LOOCV;
import java.io.IOException;
import java.text.ParseException;
@@ -39,9 +41,6 @@ import org.apache.hadoop.io.Text;
import org.junit.Assert;
import org.junit.Test;
-import smile.math.Math;
-import smile.validation.LOOCV;
-
public class RegressionTreeTest {
private static final boolean DEBUG = false;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java b/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
deleted file mode 100644
index e755e26..0000000
--- a/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.statistics;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import hivemall.TestUtils;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.generic.Collector;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class MovingAverageUDTFTest {
-
- @Test
- public void test() throws HiveException {
- MovingAverageUDTF udtf = new MovingAverageUDTF();
-
- ObjectInspector argOI0 = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
- ObjectInspector argOI1 = ObjectInspectorUtils.getConstantObjectInspector(
- PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3);
-
- final List<Double> results = new ArrayList<>();
- udtf.initialize(new ObjectInspector[] {argOI0, argOI1});
- udtf.setCollector(new Collector() {
- @Override
- public void collect(Object input) throws HiveException {
- Object[] objs = (Object[]) input;
- Assert.assertEquals(1, objs.length);
- Assert.assertTrue(objs[0] instanceof DoubleWritable);
- double x = ((DoubleWritable) objs[0]).get();
- results.add(x);
- }
- });
-
- udtf.process(new Object[] {1.f, null});
- udtf.process(new Object[] {2.f, null});
- udtf.process(new Object[] {3.f, null});
- udtf.process(new Object[] {4.f, null});
- udtf.process(new Object[] {5.f, null});
- udtf.process(new Object[] {6.f, null});
- udtf.process(new Object[] {7.f, null});
-
- Assert.assertEquals(Arrays.asList(1.d, 1.5d, 2.d, 3.d, 4.d, 5.d, 6.d), results);
- }
-
- @Test
- public void testSerialization() throws HiveException {
- TestUtils.testGenericUDTFSerialization(MovingAverageUDTF.class,
- new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
- ObjectInspectorUtils.getConstantObjectInspector(
- PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3)},
- new Object[][] {{1.f}, {2.f}, {3.f}, {4.f}, {5.f}});
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/GenerateSeriesUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/GenerateSeriesUDTFTest.java b/core/src/test/java/hivemall/tools/GenerateSeriesUDTFTest.java
new file mode 100644
index 0000000..04f432c
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/GenerateSeriesUDTFTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools;
+
+import hivemall.TestUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GenerateSeriesUDTFTest {
+
+ @Test
+ public void testTwoConstArgs() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+ TypeInfoFactory.intTypeInfo, new IntWritable(1)),
+ PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+ TypeInfoFactory.intTypeInfo, new IntWritable(3))});
+
+ final List<IntWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ IntWritable row0 = (IntWritable) row[0];
+ actual.add(new IntWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {new IntWritable(1), new IntWritable(3)});
+
+ List<IntWritable> expected =
+ Arrays.asList(new IntWritable(1), new IntWritable(2), new IntWritable(3));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testTwoIntArgs() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector});
+
+ final List<IntWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ IntWritable row0 = (IntWritable) row[0];
+ actual.add(new IntWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {1, new IntWritable(3)});
+
+ List<IntWritable> expected =
+ Arrays.asList(new IntWritable(1), new IntWritable(2), new IntWritable(3));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testTwoLongArgs() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableLongObjectInspector});
+
+ final List<LongWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ LongWritable row0 = (LongWritable) row[0];
+ actual.add(new LongWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {1, new LongWritable(3)});
+
+ List<LongWritable> expected =
+ Arrays.asList(new LongWritable(1), new LongWritable(2), new LongWritable(3));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testThreeIntArgs() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaLongObjectInspector});
+
+ final List<IntWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ IntWritable row0 = (IntWritable) row[0];
+ actual.add(new IntWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {1, new IntWritable(7), 3L});
+
+ List<IntWritable> expected =
+ Arrays.asList(new IntWritable(1), new IntWritable(4), new IntWritable(7));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testThreeLongArgs() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaLongObjectInspector,
+ PrimitiveObjectInspectorFactory.writableLongObjectInspector,
+ PrimitiveObjectInspectorFactory.javaLongObjectInspector});
+
+ final List<LongWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ LongWritable row0 = (LongWritable) row[0];
+ actual.add(new LongWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {1L, new LongWritable(7), 3L});
+
+ List<LongWritable> expected =
+ Arrays.asList(new LongWritable(1), new LongWritable(4), new LongWritable(7));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testNegativeStepInt() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaLongObjectInspector});
+
+ final List<IntWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ IntWritable row0 = (IntWritable) row[0];
+ actual.add(new IntWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {5, new IntWritable(1), -2L});
+
+ List<IntWritable> expected =
+ Arrays.asList(new IntWritable(5), new IntWritable(3), new IntWritable(1));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testNegativeStepLong() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaLongObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector});
+
+ final List<LongWritable> actual = new ArrayList<>();
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {
+ Object[] row = (Object[]) args;
+ LongWritable row0 = (LongWritable) row[0];
+ actual.add(new LongWritable(row0.get()));
+ }
+ });
+
+ udtf.process(new Object[] {5L, new IntWritable(1), -2});
+
+ List<LongWritable> expected =
+ Arrays.asList(new LongWritable(5), new LongWritable(3), new LongWritable(1));
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testSerialization() throws HiveException {
+ GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();
+
+ udtf.initialize(
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector});
+
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object args) throws HiveException {}
+ });
+
+ udtf.process(new Object[] {1, new IntWritable(3)});
+
+ byte[] serialized = TestUtils.serializeObjectByKryo(udtf);
+ TestUtils.deserializeObjectByKryo(serialized, GenerateSeriesUDTF.class);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
index b376abe..1e01274 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
@@ -85,22 +85,21 @@ public class ArrayAppendUDFTest {
udf.close();
}
-
@Test
- public void testEvaluateReturnNull() throws HiveException, IOException {
+ public void testEvaluateNullList() throws HiveException, IOException {
ArrayAppendUDF udf = new ArrayAppendUDF();
udf.initialize(new ObjectInspector[] {
ObjectInspectorFactory.getStandardListObjectInspector(
- PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});
DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
- new GenericUDF.DeferredJavaObject(new Double(3))};
+ new GenericUDF.DeferredJavaObject(new Double(3d))};
List<Object> result = udf.evaluate(args);
- Assert.assertNull(result);
+ Assert.assertEquals(Arrays.asList(new DoubleWritable(3d)), result);
udf.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
index f69cdd8..11754aa 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
@@ -18,11 +18,12 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -64,4 +65,5 @@ public class ArrayFlattenUDFTest {
new Object[] {Arrays.asList(Arrays.asList(0, 1, 2, 3), Arrays.asList(4, 5),
Arrays.asList(6, 7))});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java b/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
index a260936..fbc212a 100644
--- a/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
@@ -18,12 +18,13 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -129,4 +130,5 @@ public class ArraySliceUDFTest {
new Object[] {Arrays.asList("zero", "one", "two", "three", "four", "five", "six",
"seven", "eight", "nine", "ten"), 2, 5});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ArrayToStrUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayToStrUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayToStrUDFTest.java
new file mode 100644
index 0000000..373217f
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/array/ArrayToStrUDFTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.array;
+
+import hivemall.TestUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ArrayToStrUDFTest {
+
+ @Test
+ public void testSimpleCase() throws HiveException, IOException {
+ ArrayToStrUDF udf = new ArrayToStrUDF();
+
+ udf.initialize(new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector),
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector});
+
+ Text sep = new Text("#");
+ DeferredObject[] args =
+ new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
+ new GenericUDF.DeferredJavaObject(sep)};
+ Assert.assertEquals("1#2#3", udf.evaluate(args));
+
+ args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
+ new GenericUDF.DeferredJavaObject(null)};
+ Assert.assertEquals("1,2,3", udf.evaluate(args));
+
+ udf.close();
+ }
+
+ @Test
+ public void testNoSep() throws HiveException, IOException {
+ ArrayToStrUDF udf = new ArrayToStrUDF();
+
+ udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector)});
+
+ DeferredObject[] args =
+ new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3))};
+
+ Assert.assertEquals("1,2,3", udf.evaluate(args));
+
+ udf.close();
+ }
+
+ @Test
+ public void testNull() throws HiveException, IOException {
+ ArrayToStrUDF udf = new ArrayToStrUDF();
+
+ udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector)});
+
+ DeferredObject[] args =
+ new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, null, 3))};
+
+ Assert.assertEquals("1,3", udf.evaluate(args));
+
+ args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(null, 2, 3))};
+
+ Assert.assertEquals("2,3", udf.evaluate(args));
+
+ udf.close();
+ }
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(ArrayToStrUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector),
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector},
+ new Object[] {Arrays.asList(1, 2, 3), "-"});
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
index ac0f735..cc17039 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
@@ -74,4 +74,5 @@ public class ArrayUnionUDFTest {
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
new Object[] {Arrays.asList(0.d, 1.d), Arrays.asList(2.d, 3.d)});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
index ef45983..7045235 100644
--- a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
@@ -18,12 +18,12 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.Collector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -80,4 +80,5 @@ public class ConditionalEmitUDTFTest {
{Arrays.asList(true, false, true), Arrays.asList("one", "two", "three")},
{Arrays.asList(true, true, false), Arrays.asList("one", "two", "three")}});
}
+
}
[2/4] incubator-hivemall git commit: [HIVEMALL-145] Merge Brickhouse
functions
Posted by my...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/datetime/SessionizeUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/datetime/SessionizeUDFTest.java b/core/src/test/java/hivemall/tools/datetime/SessionizeUDFTest.java
new file mode 100644
index 0000000..2aca351
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/datetime/SessionizeUDFTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.datetime;
+
+import static hivemall.utils.hadoop.WritableUtils.val;
+
+import hivemall.TestUtils;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SessionizeUDFTest {
+
+ @Test
+ public void testTwoArgs() {
+ SessionizeUDF udf = new SessionizeUDF();
+
+ Text session1 = new Text(udf.evaluate(val(30L), val(10L)));
+ Assert.assertNotNull(session1);
+
+ Text session2 = new Text(udf.evaluate(val(35L), val(10L)));
+ Assert.assertEquals(session1, session2);
+
+ Text session3 = new Text(udf.evaluate(val(40L), val(10L)));
+ Assert.assertEquals(session2, session3);
+
+ Text session4 = new Text(udf.evaluate(val(50L), val(10L)));
+ Assert.assertNotEquals(session3, session4);
+ }
+
+ @Test
+ public void testThreeArgs() {
+ SessionizeUDF udf = new SessionizeUDF();
+
+ Text session1 = new Text(udf.evaluate(val(30L), val(10L), val("subject1")));
+ Assert.assertNotNull(session1);
+
+ Text session2 = new Text(udf.evaluate(val(35L), val(10L), val("subject1")));
+ Assert.assertEquals(session1, session2);
+
+ Text session3 = new Text(udf.evaluate(val(40L), val(10L), val("subject2")));
+ Assert.assertNotEquals(session2, session3);
+
+ Text session4 = new Text(udf.evaluate(val(45L), val(10L), val("subject2")));
+ Assert.assertEquals(session3, session4);
+ }
+
+ @Test
+ public void testSerialization() throws HiveException {
+ SessionizeUDF udf = new SessionizeUDF();
+
+ udf.evaluate(val((long) (System.currentTimeMillis() / 1000.0d)), val(30L));
+ udf.evaluate(val((long) (System.currentTimeMillis() / 1000.0d)), val(30L));
+
+ byte[] serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, SessionizeUDF.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
index 8bb8db7..738a939 100644
--- a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
@@ -90,4 +90,5 @@ public class FromJsonUDFTest {
HiveUtils.getConstStringObjectInspector("array<double>")},
new Object[] {"[0.1,1.1,2.2]"});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
index 39bd64f..f7f698c 100644
--- a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
@@ -21,6 +21,9 @@ package hivemall.tools.json;
import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
+import java.io.IOException;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -31,9 +34,6 @@ import org.apache.hadoop.io.Text;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.Arrays;
-
public class ToJsonUDFTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/map/MapKeyValuesUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/map/MapKeyValuesUDFTest.java b/core/src/test/java/hivemall/tools/map/MapKeyValuesUDFTest.java
new file mode 100644
index 0000000..2164dc1
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/map/MapKeyValuesUDFTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.map;
+
+import hivemall.TestUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MapKeyValuesUDFTest {
+
+
+ @Test
+ public void testStringDouble() throws HiveException, IOException {
+ MapKeyValuesUDF udf = new MapKeyValuesUDF();
+
+ udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});
+
+ Map<String, DoubleWritable> input = new HashMap<>();
+ for (int i = 0; i < 10; i++) {
+ input.put("k" + i, new DoubleWritable(i));
+ }
+
+ GenericUDF.DeferredObject[] arguments =
+ new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
+
+ List<Object[]> actual = udf.evaluate(arguments);
+
+ Assert.assertEquals(input.size(), actual.size());
+ for (Object[] e : actual) {
+ Assert.assertEquals(2, e.length);
+ Object v = input.get(e[0]);
+ Assert.assertEquals(e[1], v);
+ }
+
+ udf.close();
+ }
+
+ @Test
+ public void testSerialization() throws UDFArgumentException {
+ MapKeyValuesUDF udf = new MapKeyValuesUDF();
+
+ udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});
+
+ byte[] serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, MapKeyValuesUDF.class);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/sanity/RaiseErrorUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/sanity/RaiseErrorUDFTest.java b/core/src/test/java/hivemall/tools/sanity/RaiseErrorUDFTest.java
index 004ba26..a96ea57 100644
--- a/core/src/test/java/hivemall/tools/sanity/RaiseErrorUDFTest.java
+++ b/core/src/test/java/hivemall/tools/sanity/RaiseErrorUDFTest.java
@@ -18,15 +18,21 @@
*/
package hivemall.tools.sanity;
+import java.io.IOException;
+
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
import org.junit.Test;
public class RaiseErrorUDFTest {
@Test(expected = HiveException.class)
- public void test() throws HiveException {
+ public void test() throws HiveException, IOException {
RaiseErrorUDF udf = new RaiseErrorUDF();
- udf.evaluate();
+
+ udf.evaluate(new DeferredObject[] {});
+
+ udf.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/timeseries/MovingAverageUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/timeseries/MovingAverageUDTFTest.java b/core/src/test/java/hivemall/tools/timeseries/MovingAverageUDTFTest.java
new file mode 100644
index 0000000..815b567
--- /dev/null
+++ b/core/src/test/java/hivemall/tools/timeseries/MovingAverageUDTFTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.timeseries;
+
+import hivemall.TestUtils;
+import hivemall.tools.timeseries.MovingAverageUDTF;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MovingAverageUDTFTest {
+
+ @Test
+ public void test() throws HiveException {
+ MovingAverageUDTF udtf = new MovingAverageUDTF();
+
+ ObjectInspector argOI0 = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
+ ObjectInspector argOI1 = ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3);
+
+ final List<Double> results = new ArrayList<>();
+ udtf.initialize(new ObjectInspector[] {argOI0, argOI1});
+ udtf.setCollector(new Collector() {
+ @Override
+ public void collect(Object input) throws HiveException {
+ Object[] objs = (Object[]) input;
+ Assert.assertEquals(1, objs.length);
+ Assert.assertTrue(objs[0] instanceof DoubleWritable);
+ double x = ((DoubleWritable) objs[0]).get();
+ results.add(x);
+ }
+ });
+
+ udtf.process(new Object[] {1.f, null});
+ udtf.process(new Object[] {2.f, null});
+ udtf.process(new Object[] {3.f, null});
+ udtf.process(new Object[] {4.f, null});
+ udtf.process(new Object[] {5.f, null});
+ udtf.process(new Object[] {6.f, null});
+ udtf.process(new Object[] {7.f, null});
+
+ Assert.assertEquals(Arrays.asList(1.d, 1.5d, 2.d, 3.d, 4.d, 5.d, 6.d), results);
+ }
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(MovingAverageUDTF.class,
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3)},
+ new Object[][] {{1.f}, {2.f}, {3.f}, {4.f}, {5.f}});
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
index 0aa90e7..fd70fcb 100644
--- a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
+++ b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
@@ -94,4 +94,5 @@ public class VectorAddUDFTest {
PrimitiveObjectInspectorFactory.javaFloatObjectInspector)},
new Object[] {Arrays.asList(1.d, 2.d, 3.d), Arrays.asList(2.f, 3.f, 4.f)});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
index b13c447..eb5c08f 100644
--- a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
+++ b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
@@ -52,8 +52,8 @@ public class VectorDotUDFTest {
new GenericUDF.DeferredJavaObject(
WritableUtils.toWritableList(new float[] {2, 3, 4}))};
- List<Double> actual = udf.evaluate(args);
- List<Double> expected = Arrays.asList(2.d, 6.d, 12.d);
+ Object actual = udf.evaluate(args);
+ Double expected = Double.valueOf(1.d * 2.d + 2.d * 3.d + 3.d * 4.d);
Assert.assertEquals(expected, actual);
@@ -74,7 +74,7 @@ public class VectorDotUDFTest {
WritableUtils.toWritableList(new double[] {1, 2, 3})),
new GenericUDF.DeferredJavaObject(WritableUtils.val(2.f))};
- List<Double> actual = udf.evaluate(args);
+ Object actual = udf.evaluate(args);
List<Double> expected = Arrays.asList(2.d, 4.d, 6.d);
Assert.assertEquals(expected, actual);
@@ -92,4 +92,5 @@ public class VectorDotUDFTest {
PrimitiveObjectInspectorFactory.javaFloatObjectInspector)},
new Object[] {Arrays.asList(1.d, 2.d, 3.d), Arrays.asList(2.f, 3.f, 4.f)});
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/docs/gitbook/misc/funcs.md
----------------------------------------------------------------------
diff --git a/docs/gitbook/misc/funcs.md b/docs/gitbook/misc/funcs.md
index 00d7bba..3449419 100644
--- a/docs/gitbook/misc/funcs.md
+++ b/docs/gitbook/misc/funcs.md
@@ -393,6 +393,84 @@ This page describes a list of Hivemall functions. See also a [list of generic Hi
- `approx_count_distinct(expr x [, const string options])` - Returns an approximation of count(DISTINCT x) using HyperLogLogPlus algorithm
+- `bloom(string key)` - Constructs a BloomFilter by aggregating a set of keys
+ ```sql
+ CREATE TABLE satisfied_movies AS
+ SELECT bloom(movieid) as movies
+ FROM (
+ SELECT movieid
+ FROM ratings
+ GROUP BY movieid
+ HAVING avg(rating) >= 4.0
+ ) t;
+ ```
+
+- `bloom_and(string bloom1, string bloom2)` - Returns the logical AND of two bloom filters
+ ```sql
+ SELECT bloom_and(bf1, bf2) FROM xxx;
+ ```
+
+- `bloom_contains(string bloom, string key)` or _FUNC_(string bloom, array<string> keys) - Returns true if the bloom filter contains all the given key(s). Returns false if key is null.
+ ```sql
+ WITH satisfied_movies as (
+ SELECT bloom(movieid) as movies
+ FROM (
+ SELECT movieid
+ FROM ratings
+ GROUP BY movieid
+ HAVING avg(rating) >= 4.0
+ ) t
+ )
+ SELECT
+ l.rating,
+ count(distinct l.userid) as cnt
+ FROM
+ ratings l
+ CROSS JOIN satisfied_movies r
+ WHERE
+ bloom_contains(r.movies, l.movieid) -- includes false positive
+ GROUP BY
+ l.rating;
+
+ l.rating cnt
+ 1 1296
+ 2 2770
+ 3 5008
+ 4 5824
+ 5 5925
+ ```
+
+- `bloom_contains_any(string bloom, string key)` or _FUNC_(string bloom, array<string> keys)- Returns true if the bloom filter contains any of the given key
+ ```sql
+ WITH data1 as (
+ SELECT explode(array(1,2,3,4,5)) as id
+ ),
+ data2 as (
+ SELECT explode(array(1,3,5,6,8)) as id
+ ),
+ bloom as (
+ SELECT bloom(id) as bf
+ FROM data1
+ )
+ SELECT
+ l.*
+ FROM
+ data2 l
+ CROSS JOIN bloom r
+ WHERE
+ bloom_contains_any(r.bf, array(l.id))
+ ```
+
+- `bloom_not(string bloom)` - Returns the logical NOT of a bloom filters
+ ```sql
+ SELECT bloom_not(bf) FROM xxx;
+ ```
+
+- `bloom_or(string bloom1, string bloom2)` - Returns the logical OR of two bloom filters
+ ```sql
+ SELECT bloom_or(bf1, bf2) FROM xxx;
+ ```
+
# Ensemble learning
- `argmin_kld(float mean, float covar)` - Returns mean or covar that minimize a KL-distance among distributions
@@ -446,7 +524,7 @@ This page describes a list of Hivemall functions. See also a [list of generic Hi
- `hivemall_version()` - Returns the version of Hivemall
```sql
- Usage: SELECT hivemall_version();
+ SELECT hivemall_version();
```
- `lr_datagen(options string)` - Generates a logistic regression dataset
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/docs/gitbook/misc/generic_funcs.md
----------------------------------------------------------------------
diff --git a/docs/gitbook/misc/generic_funcs.md b/docs/gitbook/misc/generic_funcs.md
index d33ab21..343a64a 100644
--- a/docs/gitbook/misc/generic_funcs.md
+++ b/docs/gitbook/misc/generic_funcs.md
@@ -24,88 +24,183 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
# Array
- `array_append(array<T> arr, T elem)` - Append an element to the end of an array
+ ```sql
+ SELECT array_append(array(1,2),3);
+ 1,2,3
+
+ SELECT array_append(array('a','b'),'c');
+ "a","b","c"
+ ```
- `array_avg(array<number>)` - Returns an array<double> in which each element is the mean of a set of numbers
- `array_concat(array<ANY> x1, array<ANY> x2, ..)` - Returns a concatenated array
```sql
- select array_concat(array(1),array(2,3));
- > [1,2,3]
+ SELECT array_concat(array(1),array(2,3));
+ [1,2,3]
```
- `array_flatten(array<array<ANY>>)` - Returns an array with the elements flattened.
+ ```sql
+ SELECT array_flatten(array(array(1,2,3),array(4,5),array(6,7,8)));
+ [1,2,3,4,5,6,7,8]
+ ```
- `array_intersect(array<ANY> x1, array<ANY> x2, ..)` - Returns an intersect of given arrays
```sql
- select array_intersect(array(1,3,4),array(2,3,4),array(3,5));
- > [3]
+ SELECT array_intersect(array(1,3,4),array(2,3,4),array(3,5));
+ [3]
```
- `array_remove(array<int|text> original, int|text|array<int> target)` - Returns an array that the target is removed from the original array
```sql
- select array_remove(array(1,null,3),array(null));
- > [3]
+ SELECT array_remove(array(1,null,3),array(null));
+ [3]
- select array_remove(array("aaa","bbb"),"bbb");
- > ["aaa"]
+ SELECT array_remove(array("aaa","bbb"),"bbb");
+ ["aaa"]
```
- `array_slice(array<ANY> values, int offset [, int length])` - Slices the given array by the given offset and length parameters.
```sql
- select array_slice(array(1,2,3,4,5,6), 2,4);
- > [3,4]
+ SELECT
+ array_slice(array(1,2,3,4,5,6), 2,4),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ 0, -- offset
+ 2 -- length
+ ),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ 6, -- offset
+ 3 -- length
+ ),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ 6, -- offset
+ 10 -- length
+ ),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ 6 -- offset
+ ),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ -3 -- offset
+ ),
+ array_slice(
+ array("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"),
+ -3, -- offset
+ 2 -- length
+ );
+
+ [3,4]
+ ["zero","one"]
+ ["six","seven","eight"]
+ ["six","seven","eight","nine","ten"]
+ ["six","seven","eight","nine","ten"]
+ ["eight","nine","ten"]
+ ["eight","nine"]
```
- `array_sum(array<number>)` - Returns an array<double> in which each element is summed up
+- `array_to_str(array arr [, string sep=','])` - Convert array to string using a sperator
+ ```sql
+ SELECT array_to_str(array(1,2,3),'-');
+ 1-2-3
+ ```
+
- `array_union(array1, array2, ...)` - Returns the union of a set of arrays
+ ```sql
+ SELECT array_union(array(1,2),array(1,2));
+ [1,2]
+
+ SELECT array_union(array(1,2),array(2,3),array(2,5));
+ [1,2,3,5]
+ ```
- `conditional_emit(array<boolean> conditions, array<primitive> features)` - Emit features of a row according to various conditions
+ ```sql
+ WITH input as (
+ select array(true, false, true) as conditions, array("one", "two", "three") as features
+ UNION ALL
+ select array(true, true, false), array("four", "five", "six")
+ )
+ SELECT
+ conditional_emit(
+ conditions, features
+ )
+ FROM
+ input;
+ one
+ three
+ four
+ five
+ ```
- `element_at(array<T> list, int pos)` - Returns an element at the given position
+ ```sql
+ SELECT element_at(array(1,2,3,4),0);
+ 1
+
+ SELECT element_at(array(1,2,3,4),-2);
+ 3
+ ```
+
+- `first_element(x)` - Returns the first element in an array
+ ```sql
+ SELECT first_element(array('a','b','c'));
+ a
-- `first_element(x)` - Returns the first element in an array
+ SELECT first_element(array());
+ NULL
+ ```
- `float_array(nDims)` - Returns an array<float> of nDims elements
- `last_element(x)` - Return the last element in an array
+ ```sql
+ SELECT last_element(array('a','b','c'));
+ c
+ ```
- `select_k_best(array<number> array, const array<number> importance, const int k)` - Returns selected top-k elements as array<double>
- `sort_and_uniq_array(array<int>)` - Takes array<int> and returns a sorted array with duplicate elements eliminated
```sql
- select sort_and_uniq_array(array(3,1,1,-2,10));
- > [-2,1,3,10]
+ SELECT sort_and_uniq_array(array(3,1,1,-2,10));
+ [-2,1,3,10]
```
- `subarray_endwith(array<int|text> original, int|text key)` - Returns an array that ends with the specified key
```sql
- select subarray_endwith(array(1,2,3,4), 3);
- > [1,2,3]
+ SELECT subarray_endwith(array(1,2,3,4), 3);
+ [1,2,3]
```
- `subarray_startwith(array<int|text> original, int|text key)` - Returns an array that starts with the specified key
```sql
- select subarray_startwith(array(1,2,3,4), 2);
- > [2,3,4]
+ SELECT subarray_startwith(array(1,2,3,4), 2);
+ [2,3,4]
```
- `to_string_array(array<ANY>)` - Returns an array of strings
- `to_ordered_list(PRIMITIVE value [, PRIMITIVE key, const string options])` - Return list of values sorted by value itself or specific key
```sql
- with t as (
- select 5 as key, 'apple' as value
- union all
- select 3 as key, 'banana' as value
- union all
- select 4 as key, 'candy' as value
- union all
- select 2 as key, 'donut' as value
- union all
- select 3 as key, 'egg' as value
+ WITH t as (
+ SELECT 5 as key, 'apple' as value
+ UNION ALL
+ SELECT 3 as key, 'banana' as value
+ UNION ALL
+ SELECT 4 as key, 'candy' as value
+ UNION ALL
+ SELECT 2 as key, 'donut' as value
+ UNION ALL
+ SELECT 3 as key, 'egg' as value
)
- select -- expected output
+ SELECT -- expected output
to_ordered_list(value, key, '-reverse'), -- [apple, candy, (banana, egg | egg, banana), donut] (reverse order)
to_ordered_list(value, key, '-k 2'), -- [apple, candy] (top-k)
to_ordered_list(value, key, '-k 100'), -- [apple, candy, (banana, egg | egg, banana), dunut]
@@ -117,16 +212,230 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
to_ordered_list(value, '-k 2'), -- [egg, donut] (alphabetically)
to_ordered_list(key, '-k -2 -reverse'), -- [5, 4] (top-2 keys)
to_ordered_list(key) -- [2, 3, 3, 4, 5] (natural ordered keys)
- from
+ FROM
t
```
+# Bitset
+
+- `bits_collect(int|long x)` - Returns a bitset in array<long>
+
+- `bits_or(array<long> b1, array<long> b2, ..)` - Returns a logical OR given bitsets
+ ```sql
+ SELECT unbits(bits_or(to_bits(array(1,4)),to_bits(array(2,3))));
+ [1,2,3,4]
+ ```
+
+- `to_bits(int[] indexes)` - Returns an bitset representation if the given indexes in long[]
+ ```sql
+ SELECT to_bits(array(1,2,3,128));
+ [14,-9223372036854775808]
+ ```
+
+- `unbits(long[] bitset)` - Returns an long array of the give bitset representation
+ ```sql
+ SELECT unbits(to_bits(array(1,4,2,3)));
+ [1,2,3,4]
+ ```
+
+# Compression
+
+- `deflate(TEXT data [, const int compressionLevel])` - Returns a compressed BINARY object by using Deflater. The compression level must be in range [-1,9]
+ ```sql
+ SELECT base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));
+ AA+=kaIM|WTt!+wbGAA
+ ```
+
+- `inflate(BINARY compressedData)` - Returns a decompressed STRING by using Inflater
+ ```sql
+ SELECT inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));
+ aaaaaaaaaaaaaaaabbbbccc
+ ```
+
+# Datetime
+
+- `sessionize(long timeInSec, long thresholdInSec [, String subject])`- Returns a UUID string of a session.
+ ```sql
+ SELECT
+ sessionize(time, 3600, ip_addr) as session_id,
+ time, ip_addr
+ FROM (
+ SELECT time, ipaddr
+ FROM weblog
+ DISTRIBUTE BY ip_addr, time SORT BY ip_addr, time DESC
+ ) t1
+ ```
+
+# JSON
+
+- `from_json(string jsonString, const string returnTypes [, const array<string>|const string columnNames])` - Return Hive object.
+ ```sql
+ SELECT
+ from_json(
+ '{ "person" : { "name" : "makoto" , "age" : 37 } }',
+ 'struct<name:string,age:int>',
+ array('person')
+ ),
+ from_json(
+ '[0.1,1.1,2.2]',
+ 'array<double>'
+ ),
+ from_json(to_json(
+ ARRAY(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ NAMED_STRUCT("country", "japan", "city", "osaka")
+ )
+ ),'array<struct<country:string,city:string>>'),
+ from_json(to_json(
+ ARRAY(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ NAMED_STRUCT("country", "japan", "city", "osaka")
+ ),
+ array('city')
+ ), 'array<struct<country:string,city:string>>'),
+ from_json(to_json(
+ ARRAY(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ NAMED_STRUCT("country", "japan", "city", "osaka")
+ )
+ ),'array<struct<city:string>>');
+ ```
+
+ ```
+ {"name":"makoto","age":37}
+ [0.1,1.1,2.2]
+ [{"country":"japan","city":"tokyo"},{"country":"japan","city":"osaka"}]
+ [{"country":"japan","city":"tokyo"},{"country":"japan","city":"osaka"}]
+ [{"city":"tokyo"},{"city":"osaka"}]
+ ```
+
+- `to_json(ANY object [, const array<string>|const string columnNames])` - Returns Json string
+ ```sql
+ SELECT
+ NAMED_STRUCT("Name", "John", "age", 31),
+ to_json(
+ NAMED_STRUCT("Name", "John", "age", 31)
+ ),
+ to_json(
+ NAMED_STRUCT("Name", "John", "age", 31),
+ array('Name', 'age')
+ ),
+ to_json(
+ NAMED_STRUCT("Name", "John", "age", 31),
+ array('name', 'age')
+ ),
+ to_json(
+ NAMED_STRUCT("Name", "John", "age", 31),
+ array('age')
+ ),
+ to_json(
+ NAMED_STRUCT("Name", "John", "age", 31),
+ array()
+ ),
+ to_json(
+ null,
+ array()
+ ),
+ to_json(
+ struct("123", "456", 789, array(314,007)),
+ array('ti','si','i','bi')
+ ),
+ to_json(
+ struct("123", "456", 789, array(314,007)),
+ 'ti,si,i,bi'
+ ),
+ to_json(
+ struct("123", "456", 789, array(314,007))
+ ),
+ to_json(
+ NAMED_STRUCT("country", "japan", "city", "tokyo")
+ ),
+ to_json(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ array('city')
+ ),
+ to_json(
+ ARRAY(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ NAMED_STRUCT("country", "japan", "city", "osaka")
+ )
+ ),
+ to_json(
+ ARRAY(
+ NAMED_STRUCT("country", "japan", "city", "tokyo"),
+ NAMED_STRUCT("country", "japan", "city", "osaka")
+ ),
+ array('city')
+ );
+ ```
+
+ ```
+ {"name":"John","age":31}
+ {"name":"John","age":31}
+ {"Name":"John","age":31}
+ {"name":"John","age":31}
+ {"age":31}
+ {}
+ NULL
+ {"ti":"123","si":"456","i":789,"bi":[314,7]}
+ {"ti":"123","si":"456","i":789,"bi":[314,7]}
+ {"col1":"123","col2":"456","col3":789,"col4":[314,7]}
+ {"country":"japan","city":"tokyo"}
+ {"city":"tokyo"}
+ [{"country":"japan","city":"tokyo"},{"country":"japan","city":"osaka"}]
+ [{"country":"japan","city":"tokyo"},{"country":"japan","city":"osaka"}]
+ ```
+
# Map
+- `map_exclude_keys(Map<K,V> map, array<K> filteringKeys)` - Returns the filtered entries of a map not having specified keys
+ ```sql
+ SELECT map_exclude_keys(map(1,'one',2,'two',3,'three'),array(2,3));
+ {1:"one"}
+ ```
+
- `map_get_sum(map<int,float> src, array<int> keys)` - Returns sum of values that are retrieved by keys
+- `map_include_keys(Map<K,V> map, array<K> filteringKeys)` - Returns the filtered entries of a map having specified keys
+ ```sql
+ SELECT map_include_keys(map(1,'one',2,'two',3,'three'),array(2,3));
+ {2:"two",3:"three"}
+ ```
+
+- `map_index(a, n)` - Returns the n-th element of the given array
+ ```sql
+ WITH tmp as (
+ SELECT "one" as key
+ UNION ALL
+ SELECT "two" as key
+ )
+ SELECT map_index(map("one",1,"two",2),key)
+ FROM tmp;
+
+ 1
+ 2
+ ```
+
+- `map_key_values(map)` - Returns a array of key-value pairs.
+ ```sql
+ SELECT map_key_values(map("one",1,"two",2));
+
+ [{"key":"one","value":1},{"key":"two","value":2}]
+ ```
+
- `map_tail_n(map SRC, int N)` - Returns the last N elements from a sorted array of SRC
+- `merge_maps(x)` - Returns a map which contains the union of an aggregation of maps. Note that an existing value of a key can be replaced with the other duplicate key entry.
+ ```sql
+ SELECT
+ merge_maps(m)
+ FROM (
+ SELECT map('A',10,'B',20,'C',30)
+ UNION ALL
+ SELECT map('A',10,'B',20,'C',30)
+ ) t
+ ```
+
- `to_map(key, value)` - Convert two aggregated columns into a key-value map
- `to_ordered_map(key, value [, const int k|const boolean reverseOrder=false])` - Convert two aggregated columns into an ordered key-value map
@@ -152,42 +461,6 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
from t
```
-# Bitset
-
-- `bits_collect(int|long x)` - Returns a bitset in array<long>
-
-- `bits_or(array<long> b1, array<long> b2, ..)` - Returns a logical OR given bitsets
- ```sql
- select unbits(bits_or(to_bits(array(1,4)),to_bits(array(2,3))));
- > [1,2,3,4]
- ```
-
-- `to_bits(int[] indexes)` - Returns an bitset representation if the given indexes in long[]
- ```sql
- select to_bits(array(1,2,3,128));
- > [14,-9223372036854775808]
- ```
-
-- `unbits(long[] bitset)` - Returns an long array of the give bitset representation
- ```sql
- select unbits(to_bits(array(1,4,2,3)));
- > [1,2,3,4]
- ```
-
-# Compression
-
-- `deflate(TEXT data [, const int compressionLevel])` - Returns a compressed BINARY object by using Deflater. The compression level must be in range [-1,9]
- ```sql
- select base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));
- > AA+=kaIM|WTt!+wbGAA
- ```
-
-- `inflate(BINARY compressedData)` - Returns a decompressed STRING by using Inflater
- ```sql
- select inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));
- > aaaaaaaaaaaaaaaabbbbccc
- ```
-
# MapReduce
- `distcache_gets(filepath, key, default_value [, parseKey])` - Returns map<key_type, value_type>|value_type
@@ -198,9 +471,9 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
- `rowid()` - Returns a generated row id of a form {TASK_ID}-{SEQUENCE_NUMBER}
-- `rownum()` - Returns a generated row number in long
- ```
- returns sprintf(`%d%04d`,sequence,taskId) as long
+- `rownum()` - Returns a generated row number `sprintf(`%d%04d`,sequence,taskId)` in long
+ ```sql
+ SELECT rownum() as rownum, xxx from ...
```
- `taskid()` - Returns the value of mapred.task.partition
@@ -215,47 +488,91 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
- `transpose_and_dot(array<number> matrix0_row, array<number> matrix1_row)` - Returns dot(matrix0.T, matrix1) as array<array<double>>, shape = (matrix0.#cols, matrix1.#cols)
+# Sanity Checks
+
+- `assert(boolean condition)` or _FUNC_(boolean condition, string errMsg)- Throws HiveException if condition is not met
+ ```sql
+ SELECT count(1) FROM stock_price WHERE assert(price > 0.0);
+ SELECT count(1) FROM stock_price WHERE assert(price > 0.0, 'price MUST be more than 0.0')
+ ```
+
+- `raise_error()` or _FUNC_(string msg) - Throws an error
+ ```sql
+ SELECT product_id, price, raise_error('Found an invalid record') FROM xxx WHERE price < 0.0
+ ```
+
# Text processing
- `base91(BINARY bin)` - Convert the argument from binary to a BASE91 string
```sql
- select base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));
- > AA+=kaIM|WTt!+wbGAA
+ SELECT base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));
+ AA+=kaIM|WTt!+wbGAA
```
- `is_stopword(string word)` - Returns whether English stopword or not
- `normalize_unicode(string str [, string form])` - Transforms `str` with the specified normalization form. The `form` takes one of NFC (default), NFD, NFKC, or NFKD
```sql
- select normalize_unicode('ﾊﾝｶｸｶﾅ','NFKC');
- > ハンカクカナ
+ SELECT normalize_unicode('ﾊﾝｶｸｶﾅ','NFKC');
+ ハンカクカナ
- select normalize_unicode('㈱㌧㌦Ⅲ','NFKC');
- > (株)トンドルIII
+ SELECT normalize_unicode('㈱㌧㌦Ⅲ','NFKC');
+ (株)トンドルIII
```
- `singularize(string word)` - Returns singular form of a given English word
```sql
- select singularize(lower("Apples"));
+ SELECT singularize(lower("Apples"));
- > "apple"
+ "apple"
```
-- `split_words(string query [, string regex])` - Returns an array<text> containing split strings
+- `split_words(string query [, string regex])` - Returns an array<text> containing splitted strings
- `tokenize(string englishText [, boolean toLowerCase])` - Returns tokenized words in array<string>
- `unbase91(string)` - Convert a BASE91 string to a binary
```sql
- select inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));
- > aaaaaaaaaaaaaaaabbbbccc
+ SELECT inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));
+ aaaaaaaaaaaaaaaabbbbccc
```
- `word_ngrams(array<string> words, int minSize, int maxSize])` - Returns list of n-grams for given words, where `minSize <= n <= maxSize`
```sql
- select word_ngrams(tokenize('Machine learning is fun!', true), 1, 2);
+ SELECT word_ngrams(tokenize('Machine learning is fun!', true), 1, 2);
- > ["machine","machine learning","learning","learning is","is","is fun","fun"]
+ ["machine","machine learning","learning","learning is","is","is fun","fun"]
+ ```
+
+# Timeseries
+
+- `moving_avg(NUMBER value, const int windowSize)` - Returns moving average of a time series using a given window
+ ```sql
+ SELECT moving_avg(x, 3) FROM (SELECT explode(array(1.0,2.0,3.0,4.0,5.0,6.0,7.0)) as x) series;
+ 1.0
+ 1.5
+ 2.0
+ 3.0
+ 4.0
+ 5.0
+ 6.0
+ ```
+
+# Vector
+
+- `vector_add(array<NUMBER> x, array<NUMBER> y)` - Perform vector ADD operation.
+ ```sql
+ SELECT vector_add(array(1.0,2.0,3.0), array(2, 3, 4));
+ [3.0,5.0,7.0]
+ ```
+
+- `vector_dot(array<NUMBER> x, array<NUMBER> y)` - Performs vector dot product.
+ ```sql
+ SELECT vector_dot(array(1.0,2.0,3.0),array(2.0,3.0,4.0));
+ 20
+
+ SELECT vector_dot(array(1.0,2.0,3.0),2);
+ [2.0,4.0,6.0]
```
# Others
@@ -264,25 +581,51 @@ This page describes a list of useful Hivemall generic functions. See also a [lis
- `each_top_k(int K, Object group, double cmpKey, *)` - Returns top-K values (or tail-K values when k is less than 0)
-- `generate_series(const int|bigint start, const int|bigint end)` - Generate a series of values, from start to end. A similar function to PostgreSQL's `generate_serics`. http://www.postgresql.org/docs/current/static/functions-srf.html
+- `generate_series(const int|bigint start, const int|bigint end)` - Generate a series of values, from start to end. A similar function to PostgreSQL's [generate_serics](http://www.postgresql.org/docs/current/static/functions-srf.html)
```sql
- select generate_series(1,9);
+ SELECT generate_series(2,4);
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
+ 2
+ 3
+ 4
+
+ SELECT generate_series(5,1,-2);
+
+ 5
+ 3
+ 1
+
+ SELECT generate_series(4,3);
+
+ (no return)
+
+ SELECT date_add(current_date(),value),value from (SELECT generate_series(1,3)) t;
+
+ 2018-04-21 1
+ 2018-04-22 2
+ 2018-04-23 3
+
+ WITH input as (
+ SELECT 1 as c1, 10 as c2, 3 as step
+ UNION ALL
+ SELECT 10, 2, -3
+ )
+ SELECT generate_series(c1, c2, step) as series
+ FROM input;
+
+ 1
+ 4
+ 7
+ 10
+ 10
+ 7
+ 4
```
- `try_cast(ANY src, const string typeName)` - Explicitly cast a value as a type. Returns null if cast fails.
```sql
- Usage: select try_cast(array(1.0,2.0,3.0), 'array<string>')
- select try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')
+ SELECT try_cast(array(1.0,2.0,3.0), 'array<string>')
+ SELECT try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')
```
- `x_rank(KEY)` - Generates a pseudo sequence number starting from 1 for each key
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5a78bd1..aa682c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -196,6 +196,18 @@
</roles>
<timezone>+9</timezone>
</developer>
+ <developer>
+ <id>jbanks</id>
+ <name>Jerome Banks</name>
+ <email>jbanks[at]apache.org</email>
+ <url>https://github.com/jeromebanks/</url>
+ <organization>Jumpshot Inc.</organization>
+ <organizationUrl>https://www.jumpshot.com/</organizationUrl>
+ <roles>
+ <role>Committer</role>
+ </roles>
+ <timezone>-8</timezone>
+ </developer>
<!-- Project mentors -->
<developer>
<id>rvs</id>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/resources/ddl/define-all-as-permanent.hive
----------------------------------------------------------------------
diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive
index e7da8e3..5c257c5 100644
--- a/resources/ddl/define-all-as-permanent.hive
+++ b/resources/ddl/define-all-as-permanent.hive
@@ -435,6 +435,30 @@ CREATE FUNCTION array_intersect as 'hivemall.tools.array.ArrayIntersectUDF' USIN
DROP FUNCTION IF EXISTS select_k_best;
CREATE FUNCTION select_k_best as 'hivemall.tools.array.SelectKBestUDF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS array_append;
+CREATE FUNCTION array_append as 'hivemall.tools.array.ArrayAppendUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS element_at;
+CREATE FUNCTION element_at as 'hivemall.tools.array.ArrayElementAtUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS array_union;
+CREATE FUNCTION array_union as 'hivemall.tools.array.ArrayUnionUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS first_element;
+CREATE FUNCTION first_element as 'hivemall.tools.array.FirstElementUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS last_element;
+CREATE FUNCTION last_element as 'hivemall.tools.array.LastElementUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS array_flatten;
+CREATE FUNCTION array_flatten as 'hivemall.tools.array.ArrayFlattenUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS array_to_str;
+CREATE FUNCTION array_to_str as 'hivemall.tools.array.ArrayToStrUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS conditional_emit;
+CREATE FUNCTION conditional_emit as 'hivemall.tools.array.ConditionalEmitUDTF' USING JAR '${hivemall_jar}';
+
-----------------------------
-- bit operation functions --
-----------------------------
@@ -477,6 +501,18 @@ CREATE FUNCTION to_map as 'hivemall.tools.map.UDAFToMap' USING JAR '${hivemall_j
DROP FUNCTION IF EXISTS to_ordered_map;
CREATE FUNCTION to_ordered_map as 'hivemall.tools.map.UDAFToOrderedMap' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS map_include_keys;
+CREATE FUNCTION map_include_keys as 'hivemall.tools.map.MapIncludeKeysUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS map_exclude_keys;
+CREATE FUNCTION map_exclude_keys as 'hivemall.tools.map.MapExcludeKeysUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS map_index;
+CREATE FUNCTION map_index as 'hivemall.tools.map.MapIndexUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS map_key_values;
+CREATE FUNCTION map_key_values as 'hivemall.tools.map.MapKeyValuesUDF' USING JAR '${hivemall_jar}';
+
---------------------
-- list functions --
---------------------
@@ -494,13 +530,19 @@ CREATE FUNCTION sigmoid as 'hivemall.tools.math.SigmoidGenericUDF' USING JAR '${
DROP FUNCTION IF EXISTS l2_norm;
CREATE FUNCTION l2_norm as 'hivemall.tools.math.L2NormUDAF' USING JAR '${hivemall_jar}';
-----------------------
--- Matrix functions --
-----------------------
+-----------------------------
+-- Matrix/Vector functions --
+-----------------------------
DROP FUNCTION IF EXISTS transpose_and_dot;
CREATE FUNCTION transpose_and_dot as 'hivemall.tools.matrix.TransposeAndDotUDAF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS vector_add;
+CREATE FUNCTION vector_add as 'hivemall.tools.vector.VectorAddUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS vector_dot;
+CREATE FUNCTION vector_dot as 'hivemall.tools.vector.VectorDotUDF' USING JAR '${hivemall_jar}';
+
----------------------
-- mapred functions --
----------------------
@@ -524,6 +566,26 @@ DROP FUNCTION IF EXISTS jobconf_gets;
CREATE FUNCTION jobconf_gets as 'hivemall.tools.mapred.JobConfGetsUDF' USING JAR '${hivemall_jar}';
--------------------
+-- JSON functions --
+--------------------
+
+DROP FUNCTION IF EXISTS to_json;
+CREATE FUNCTION to_json as 'hivemall.tools.json.ToJsonUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS from_json;
+CREATE FUNCTION from_json as 'hivemall.tools.json.FromJsonUDF' USING JAR '${hivemall_jar}';
+
+----------------------------
+-- Sanity Check functions --
+----------------------------
+
+DROP FUNCTION IF EXISTS assert;
+CREATE FUNCTION assert as 'hivemall.tools.sanity.AssertUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS raise_error;
+CREATE FUNCTION raise_error as 'hivemall.tools.sanity.RaiseErrorUDF' USING JAR '${hivemall_jar}';
+
+--------------------
-- misc functions --
--------------------
@@ -539,6 +601,15 @@ CREATE FUNCTION x_rank as 'hivemall.tools.RankSequenceUDF' USING JAR '${hivemall
DROP FUNCTION IF EXISTS each_top_k;
CREATE FUNCTION each_top_k as 'hivemall.tools.EachTopKUDTF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS try_cast;
+CREATE FUNCTION try_cast as 'hivemall.tools.TryCastUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS sessionize;
+CREATE FUNCTION sessionize as 'hivemall.tools.datetime.SessionizeUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS moving_avg;
+CREATE FUNCTION moving_avg as 'hivemall.tools.timeseries.MovingAverageUDTF' USING JAR '${hivemall_jar}';
+
-------------------------------
-- Text processing functions --
-------------------------------
@@ -749,6 +820,28 @@ CREATE FUNCTION train_slim as 'hivemall.recommend.SlimUDTF' USING JAR '${hivemal
DROP FUNCTION IF EXISTS approx_count_distinct;
CREATE FUNCTION approx_count_distinct as 'hivemall.sketch.hll.ApproxCountDistinctUDAF' USING JAR '${hivemall_jar}';
+------------------
+-- Bloom Filter --
+------------------
+
+DROP FUNCTION IF EXISTS bloom;
+CREATE FUNCTION bloom as 'hivemall.sketch.bloom.BloomFilterUDAF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS bloom_and;
+CREATE FUNCTION bloom_and as 'hivemall.sketch.bloom.BloomAndUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS bloom_contains;
+CREATE FUNCTION bloom_contains as 'hivemall.sketch.bloom.BloomContainsUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS bloom_not;
+CREATE FUNCTION bloom_not as 'hivemall.sketch.bloom.BloomNotUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS bloom_or;
+CREATE FUNCTION bloom_or as 'hivemall.sketch.bloom.BloomOrUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS bloom_contains_any;
+CREATE FUNCTION bloom_contains_any as 'hivemall.sketch.bloom.BloomContainsAnyUDF' USING JAR '${hivemall_jar}';
+
------------------------------
-- XGBoost related features --
------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/resources/ddl/define-all.hive
----------------------------------------------------------------------
diff --git a/resources/ddl/define-all.hive b/resources/ddl/define-all.hive
index 9228ce9..fbb3ed2 100644
--- a/resources/ddl/define-all.hive
+++ b/resources/ddl/define-all.hive
@@ -427,6 +427,30 @@ create temporary function array_intersect as 'hivemall.tools.array.ArrayIntersec
drop temporary function if exists select_k_best;
create temporary function select_k_best as 'hivemall.tools.array.SelectKBestUDF';
+drop temporary function if exists array_append;
+create temporary function array_append as 'hivemall.tools.array.ArrayAppendUDF';
+
+drop temporary function if exists element_at;
+create temporary function element_at as 'hivemall.tools.array.ArrayElementAtUDF';
+
+drop temporary function if exists array_union;
+create temporary function array_union as 'hivemall.tools.array.ArrayUnionUDF';
+
+drop temporary function if exists first_element;
+create temporary function first_element as 'hivemall.tools.array.FirstElementUDF';
+
+drop temporary function if exists last_element;
+create temporary function last_element as 'hivemall.tools.array.LastElementUDF';
+
+drop temporary function if exists array_flatten;
+create temporary function array_flatten as 'hivemall.tools.array.ArrayFlattenUDF';
+
+drop temporary function if exists array_to_str;
+create temporary function array_to_str as 'hivemall.tools.array.ArrayToStrUDF';
+
+drop temporary function if exists conditional_emit;
+create temporary function conditional_emit as 'hivemall.tools.array.ConditionalEmitUDTF';
+
-----------------------------
-- bit operation functions --
-----------------------------
@@ -469,6 +493,18 @@ create temporary function to_map as 'hivemall.tools.map.UDAFToMap';
drop temporary function if exists to_ordered_map;
create temporary function to_ordered_map as 'hivemall.tools.map.UDAFToOrderedMap';
+drop temporary function if exists map_include_keys;
+create temporary function map_include_keys as 'hivemall.tools.map.MapIncludeKeysUDF';
+
+drop temporary function if exists map_exclude_keys;
+create temporary function map_exclude_keys as 'hivemall.tools.map.MapExcludeKeysUDF';
+
+drop temporary function if exists map_index;
+create temporary function map_index as 'hivemall.tools.map.MapIndexUDF';
+
+drop temporary function if exists map_key_values;
+create temporary function map_key_values as 'hivemall.tools.map.MapKeyValuesUDF';
+
---------------------
-- list functions --
---------------------
@@ -486,13 +522,19 @@ create temporary function sigmoid as 'hivemall.tools.math.SigmoidGenericUDF';
drop temporary function if exists l2_norm;
create temporary function l2_norm as 'hivemall.tools.math.L2NormUDAF';
-----------------------
--- Matrix functions --
-----------------------
+-----------------------------
+-- Matrix/Vector functions --
+-----------------------------
drop temporary function if exists transpose_and_dot;
create temporary function transpose_and_dot as 'hivemall.tools.matrix.TransposeAndDotUDAF';
+drop temporary function if exists vector_add;
+create temporary function vector_add as 'hivemall.tools.vector.VectorAddUDF';
+
+drop temporary function if exists vector_dot;
+create temporary function vector_dot as 'hivemall.tools.vector.VectorDotUDF';
+
----------------------
-- mapred functions --
----------------------
@@ -516,6 +558,26 @@ drop temporary function if exists jobconf_gets;
create temporary function jobconf_gets as 'hivemall.tools.mapred.JobConfGetsUDF';
--------------------
+-- JSON functions --
+--------------------
+
+drop temporary function if exists to_json;
+create temporary function to_json as 'hivemall.tools.json.ToJsonUDF';
+
+drop temporary function if exists from_json;
+create temporary function from_json as 'hivemall.tools.json.FromJsonUDF';
+
+----------------------------
+-- Sanity Check functions --
+----------------------------
+
+drop temporary function if exists assert;
+create temporary function assert as 'hivemall.tools.sanity.AssertUDF';
+
+drop temporary function if exists raise_error;
+create temporary function raise_error as 'hivemall.tools.sanity.RaiseErrorUDF';
+
+--------------------
-- misc functions --
--------------------
@@ -531,6 +593,15 @@ create temporary function x_rank as 'hivemall.tools.RankSequenceUDF';
drop temporary function if exists each_top_k;
create temporary function each_top_k as 'hivemall.tools.EachTopKUDTF';
+drop temporary function if exists try_cast;
+create temporary function try_cast as 'hivemall.tools.TryCastUDF';
+
+drop temporary function if exists sessionize;
+create temporary function sessionize as 'hivemall.tools.datetime.SessionizeUDF';
+
+drop temporary function if exists moving_avg;
+create temporary function moving_avg as 'hivemall.tools.timeseries.MovingAverageUDTF';
+
-------------------------------
-- Text processing functions --
-------------------------------
@@ -741,6 +812,28 @@ create temporary function train_slim as 'hivemall.recommend.SlimUDTF';
drop temporary function if exists approx_count_distinct;
create temporary function approx_count_distinct as 'hivemall.sketch.hll.ApproxCountDistinctUDAF';
+------------------
+-- Bloom Filter --
+------------------
+
+drop temporary function if exists bloom;
+create temporary function bloom as 'hivemall.sketch.bloom.BloomFilterUDAF';
+
+drop temporary function if exists bloom_and;
+create temporary function bloom_and as 'hivemall.sketch.bloom.BloomAndUDF';
+
+drop temporary function if exists bloom_contains;
+create temporary function bloom_contains as 'hivemall.sketch.bloom.BloomContainsUDF';
+
+drop temporary function if exists bloom_not;
+create temporary function bloom_not as 'hivemall.sketch.bloom.BloomNotUDF';
+
+drop temporary function if exists bloom_or;
+create temporary function bloom_or as 'hivemall.sketch.bloom.BloomOrUDF';
+
+drop temporary function if exists bloom_contains_any;
+create temporary function bloom_contains_any as 'hivemall.sketch.bloom.BloomContainsAnyUDF';
+
--------------------------------------------------------------------------------------------------
-- macros available from hive 0.12.0
-- see https://issues.apache.org/jira/browse/HIVE-2655
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/resources/ddl/define-all.spark
----------------------------------------------------------------------
diff --git a/resources/ddl/define-all.spark b/resources/ddl/define-all.spark
index 3764ca2..e78a966 100644
--- a/resources/ddl/define-all.spark
+++ b/resources/ddl/define-all.spark
@@ -425,6 +425,30 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION array_intersect AS 'hivemall.tools.arr
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS select_k_best")
sqlContext.sql("CREATE TEMPORARY FUNCTION select_k_best AS 'hivemall.tools.array.SelectKBestUDF'")
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_append")
+sqlContext.sql("CREATE TEMPORARY FUNCTION array_append AS 'hivemall.tools.array.ArrayAppendUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS element_at")
+sqlContext.sql("CREATE TEMPORARY FUNCTION element_at AS 'hivemall.tools.array.ArrayElementAtUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_union")
+sqlContext.sql("CREATE TEMPORARY FUNCTION array_union AS 'hivemall.tools.array.ArrayUnionUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS first_element")
+sqlContext.sql("CREATE TEMPORARY FUNCTION first_element AS 'hivemall.tools.array.FirstElementUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS last_element")
+sqlContext.sql("CREATE TEMPORARY FUNCTION last_element AS 'hivemall.tools.array.LastElementUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_flatten")
+sqlContext.sql("CREATE TEMPORARY FUNCTION array_flatten AS 'hivemall.tools.array.ArrayFlattenUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS array_to_str")
+sqlContext.sql("CREATE TEMPORARY FUNCTION array_to_str AS 'hivemall.tools.array.ArrayToStrUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS conditional_emit")
+sqlContext.sql("CREATE TEMPORARY FUNCTION conditional_emit AS 'hivemall.tools.array.ConditionalEmitUDTF'")
+
/**
* Bit operation functions
*/
@@ -467,6 +491,18 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION to_map AS 'hivemall.tools.map.UDAFToMa
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS to_ordered_map")
sqlContext.sql("CREATE TEMPORARY FUNCTION to_ordered_map AS 'hivemall.tools.map.UDAFToOrderedMap'")
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS map_include_keys")
+sqlContext.sql("CREATE TEMPORARY FUNCTION map_include_keys AS 'hivemall.tools.map.MapIncludeKeysUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS map_exclude_keys")
+sqlContext.sql("CREATE TEMPORARY FUNCTION map_exclude_keys AS 'hivemall.tools.map.MapExcludeKeysUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS map_index")
+sqlContext.sql("CREATE TEMPORARY FUNCTION map_index AS 'hivemall.tools.map.MapIndexUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS map_key_values")
+sqlContext.sql("CREATE TEMPORARY FUNCTION map_key_values AS 'hivemall.tools.map.MapKeyValuesUDF'")
+
/**
* List functions
*/
@@ -485,12 +521,18 @@ sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS l2_norm")
sqlContext.sql("CREATE TEMPORARY FUNCTION l2_norm AS 'hivemall.tools.math.L2NormUDAF'")
/**
- * Matrix functions
+ * Matrix/Vector functions
*/
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS transpose_and_dot")
sqlContext.sql("CREATE TEMPORARY FUNCTION transpose_and_dot AS 'hivemall.tools.matrix.TransposeAndDotUDAF'")
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS vector_add")
+sqlContext.sql("CREATE TEMPORARY FUNCTION vector_add AS 'hivemall.tools.vector.VectorAddUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS vector_dot")
+sqlContext.sql("CREATE TEMPORARY FUNCTION vector_dot AS 'hivemall.tools.vector.VectorDotUDF'")
+
/**
* MAPRED functions
*/
@@ -499,6 +541,26 @@ sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS rowid")
sqlContext.sql("CREATE TEMPORARY FUNCTION rowid AS 'hivemall.tools.mapred.RowIdUDFWrapper'")
/**
+ * JSON functions
+ */
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS to_json")
+sqlContext.sql("CREATE TEMPORARY FUNCTION to_json AS 'hivemall.tools.json.ToJsonUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS from_json")
+sqlContext.sql("CREATE TEMPORARY FUNCTION from_json AS 'hivemall.tools.json.FromJsonUDF'")
+
+/**
+ * Sanity Check functions
+ */
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS assert")
+sqlContext.sql("CREATE TEMPORARY FUNCTION assert AS 'hivemall.tools.sanity.AssertUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS raise_error")
+sqlContext.sql("CREATE TEMPORARY FUNCTION raise_error AS 'hivemall.tools.sanity.RaiseErrorUDF'")
+
+/**
* MISC functions
*/
@@ -514,6 +576,15 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION x_rank AS 'hivemall.tools.RankSequence
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS each_top_k")
sqlContext.sql("CREATE TEMPORARY FUNCTION each_top_k AS 'hivemall.tools.EachTopKUDTF'")
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS try_cast")
+sqlContext.sql("CREATE TEMPORARY FUNCTION try_cast AS 'hivemall.tools.TryCastUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS sessionize")
+sqlContext.sql("CREATE TEMPORARY FUNCTION sessionize AS 'hivemall.tools.datetime.SessionizeUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS moving_avg")
+sqlContext.sql("CREATE TEMPORARY FUNCTION moving_avg AS 'hivemall.tools.timeseries.MovingAverageUDTF'")
+
/**
* Text processing functions
*/
@@ -726,3 +797,25 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION train_slim AS 'hivemall.recommend.Slim
sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS approx_count_distinct")
sqlContext.sql("CREATE TEMPORARY FUNCTION approx_count_distinct AS 'hivemall.sketch.hll.ApproxCountDistinctUDAF'")
+
+/**
+ * Bloom Filter
+ */
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom AS 'hivemall.sketch.bloom.BloomFilterUDAF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom_and")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom_and AS 'hivemall.sketch.bloom.BloomAndUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom_contains")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom_contains AS 'hivemall.sketch.bloom.BloomContainsUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom_not")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom_not AS 'hivemall.sketch.bloom.BloomNotUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom_or")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom_or AS 'hivemall.sketch.bloom.BloomOrUDF'")
+
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS bloom_contains_any")
+sqlContext.sql("CREATE TEMPORARY FUNCTION bloom_contains_any AS 'hivemall.sketch.bloom.BloomContainsAnyUDF'")
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/resources/ddl/define-udfs.td.hql
----------------------------------------------------------------------
diff --git a/resources/ddl/define-udfs.td.hql b/resources/ddl/define-udfs.td.hql
index b106eda..2352390 100644
--- a/resources/ddl/define-udfs.td.hql
+++ b/resources/ddl/define-udfs.td.hql
@@ -155,7 +155,7 @@ create temporary function train_randomforest_regr as 'hivemall.smile.regression.
create temporary function tree_predict as 'hivemall.smile.tools.TreePredictUDF';
create temporary function rf_ensemble as 'hivemall.smile.tools.RandomForestEnsembleUDAF';
create temporary function guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF';
--- since Hivemall v0.5
+-- since Hivemall v0.5.0
create temporary function changefinder as 'hivemall.anomaly.ChangeFinderUDF';
create temporary function sst as 'hivemall.anomaly.SingularSpectrumTransformUDF';
create temporary function train_lda as 'hivemall.topicmodel.LDAUDTF';
@@ -183,7 +183,35 @@ create temporary function train_slim as 'hivemall.recommend.SlimUDTF';
create temporary function hitrate as 'hivemall.evaluation.HitRateUDAF';
create temporary function word_ngrams as 'hivemall.tools.text.WordNgramsUDF';
create temporary function approx_count_distinct as 'hivemall.sketch.hll.ApproxCountDistinctUDAF';
+-- since Hivemall v0.5.2
create temporary function array_slice as 'hivemall.tools.array.ArraySliceUDF';
+create temporary function try_cast as 'hivemall.tools.TryCastUDF';
+create temporary function array_append as 'hivemall.tools.array.ArrayAppendUDF';
+create temporary function element_at as 'hivemall.tools.array.ArrayElementAtUDF';
+create temporary function array_union as 'hivemall.tools.array.ArrayUnionUDF';
+create temporary function first_element as 'hivemall.tools.array.FirstElementUDF';
+create temporary function last_element as 'hivemall.tools.array.LastElementUDF';
+create temporary function array_flatten as 'hivemall.tools.array.ArrayFlattenUDF';
+create temporary function map_include_keys as 'hivemall.tools.map.MapIncludeKeysUDF';
+create temporary function map_exclude_keys as 'hivemall.tools.map.MapExcludeKeysUDF';
+create temporary function array_to_str as 'hivemall.tools.array.ArrayToStrUDF';
+create temporary function map_index as 'hivemall.tools.map.MapIndexUDF';
+create temporary function map_key_values as 'hivemall.tools.map.MapKeyValuesUDF';
+create temporary function sessionize as 'hivemall.tools.datetime.SessionizeUDF';
+create temporary function to_json as 'hivemall.tools.json.ToJsonUDF';
+create temporary function from_json as 'hivemall.tools.json.FromJsonUDF';
+create temporary function assert as 'hivemall.tools.sanity.AssertUDF';
+create temporary function raise_error as 'hivemall.tools.sanity.RaiseErrorUDF';
+create temporary function moving_avg as 'hivemall.tools.timeseries.MovingAverageUDTF';
+create temporary function vector_add as 'hivemall.tools.vector.VectorAddUDF';
+create temporary function vector_dot as 'hivemall.tools.vector.VectorDotUDF';
+create temporary function bloom as 'hivemall.sketch.bloom.BloomFilterUDAF';
+create temporary function bloom_and as 'hivemall.sketch.bloom.BloomAndUDF';
+create temporary function bloom_contains as 'hivemall.sketch.bloom.BloomContainsUDF';
+create temporary function bloom_not as 'hivemall.sketch.bloom.BloomNotUDF';
+create temporary function bloom_or as 'hivemall.sketch.bloom.BloomOrUDF';
+create temporary function bloom_contains_any as 'hivemall.sketch.bloom.BloomContainsAnyUDF';
+create temporary function conditional_emit as 'hivemall.tools.array.ConditionalEmitUDTF';
-- NLP features
create temporary function tokenize_ja as 'hivemall.nlp.tokenizer.KuromojiUDF';
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/spark/spark-2.2/pom.xml
----------------------------------------------------------------------
diff --git a/spark/spark-2.2/pom.xml b/spark/spark-2.2/pom.xml
index 100de59..5cb3609 100644
--- a/spark/spark-2.2/pom.xml
+++ b/spark/spark-2.2/pom.xml
@@ -141,7 +141,7 @@
<JAVA_HOME>${env.JAVA8_HOME}</JAVA_HOME>
<PATH>${env.JAVA8_HOME}/bin:${env.PATH}</PATH>
</environmentVariables>
- </configuration>
+ </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/tools/hivemall-docs/pom.xml
----------------------------------------------------------------------
diff --git a/tools/hivemall-docs/pom.xml b/tools/hivemall-docs/pom.xml
index 76994d7..99cd138 100644
--- a/tools/hivemall-docs/pom.xml
+++ b/tools/hivemall-docs/pom.xml
@@ -16,7 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -56,46 +58,94 @@
<scope>provided</scope>
</dependency>
- <!-- hivemall dependencies -->
+ <!-- compile scope -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <!-- runtime hivemall dependencies using reflection -->
<dependency>
<groupId>org.apache.hivemall</groupId>
<artifactId>hivemall-core</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.hivemall</groupId>
<artifactId>hivemall-nlp</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.hivemall</groupId>
<artifactId>hivemall-xgboost</artifactId>
<version>${project.version}</version>
- <scope>compile</scope>
+ <scope>runtime</scope>
</dependency>
- <!-- compile scope -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>0.9.10</version>
- <scope>compile</scope>
- </dependency>
</dependencies>
<build>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings
+ only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-component-metadata</artifactId>
+ <versionRange>[1.5.5,)</versionRange>
+ <goals>
+ <goal>generate-metadata</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ </execute>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>
+ org.apache.maven.plugins
+ </groupId>
+ <artifactId>
+ maven-plugin-plugin
+ </artifactId>
+ <versionRange>
+ [3.2,)
+ </versionRange>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -106,6 +156,12 @@
<phase>process-classes</phase>
</execution>
<execution>
+ <id>mojo-descriptor</id>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </execution>
+ <execution>
<id>generate-helpmojo</id>
<goals>
<goal>helpmojo</goal>
@@ -113,6 +169,19 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-component-metadata</artifactId>
+ <version>1.7.1</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate-metadata</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGenerator.java
----------------------------------------------------------------------
diff --git a/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGenerator.java b/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGenerator.java
deleted file mode 100644
index 61fea68..0000000
--- a/tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGenerator.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.docs;
-
-import static hivemall.docs.utils.MarkdownUtils.asCodeBlock;
-import static hivemall.docs.utils.MarkdownUtils.asInlineCode;
-import static hivemall.docs.utils.MarkdownUtils.asListElement;
-import static hivemall.docs.utils.MarkdownUtils.indent;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-
-import hivemall.utils.lang.StringUtils;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nonnull;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.maven.execution.MavenSession;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.reflections.Reflections;
-
-/**
- * Generate a list of UDFs for documentation.
- *
- * @link https://hivemall.incubator.apache.org/userguide/misc/generic_funcs.html
- * @link https://hivemall.incubator.apache.org/userguide/misc/funcs.html
- */
-@Mojo(name = "generate-funcs-list")
-public class FuncsListGenerator extends AbstractMojo {
-
- @Parameter(defaultValue = "${basedir}", readonly = true)
- private File basedir;
-
- @Parameter(defaultValue = "${session}", readonly = true)
- private MavenSession session;
-
- @Parameter(defaultValue = "docs/gitbook/misc/generic_funcs.md")
- private String pathToGenericFuncs;
-
- @Parameter(defaultValue = "docs/gitbook/misc/funcs.md")
- private String pathToFuncs;
-
- private static final Map<String, List<String>> genericFuncsHeaders = new LinkedHashMap<>();
- static {
- genericFuncsHeaders.put("# Array",
- Arrays.asList("hivemall.tools.array", "hivemall.tools.list"));
- genericFuncsHeaders.put("# Map", Collections.singletonList("hivemall.tools.map"));
- genericFuncsHeaders.put("# Bitset", Collections.singletonList("hivemall.tools.bits"));
- genericFuncsHeaders.put("# Compression",
- Collections.singletonList("hivemall.tools.compress"));
- genericFuncsHeaders.put("# MapReduce", Collections.singletonList("hivemall.tools.mapred"));
- genericFuncsHeaders.put("# Math", Collections.singletonList("hivemall.tools.math"));
- genericFuncsHeaders.put("# Matrix", Collections.singletonList("hivemall.tools.matrix"));
- genericFuncsHeaders.put("# Text processing",
- Collections.singletonList("hivemall.tools.text"));
- genericFuncsHeaders.put("# Others", Collections.singletonList("hivemall.tools"));
- }
-
- private static final Map<String, List<String>> funcsHeaders = new LinkedHashMap<>();
- static {
- funcsHeaders.put("# Regression", Collections.singletonList("hivemall.regression"));
- funcsHeaders.put("# Classification", Collections.<String>emptyList());
- funcsHeaders.put("## Binary classification",
- Collections.singletonList("hivemall.classifier"));
- funcsHeaders.put("## Multiclass classification",
- Collections.singletonList("hivemall.classifier.multiclass"));
- funcsHeaders.put("# Matrix factorization", Collections.singletonList("hivemall.mf"));
- funcsHeaders.put("# Factorization machines", Collections.singletonList("hivemall.fm"));
- funcsHeaders.put("# Recommendation", Collections.singletonList("hivemall.recommend"));
- funcsHeaders.put("# Anomaly detection", Collections.singletonList("hivemall.anomaly"));
- funcsHeaders.put("# Topic modeling", Collections.singletonList("hivemall.topicmodel"));
- funcsHeaders.put("# Preprocessing", Collections.singletonList("hivemall.ftvec"));
- funcsHeaders.put("## Data amplification",
- Collections.singletonList("hivemall.ftvec.amplify"));
- funcsHeaders.put("## Feature binning", Collections.singletonList("hivemall.ftvec.binning"));
- funcsHeaders.put("## Feature format conversion",
- Collections.singletonList("hivemall.ftvec.conv"));
- funcsHeaders.put("## Feature hashing", Collections.singletonList("hivemall.ftvec.hashing"));
- funcsHeaders.put("## Feature paring", Collections.singletonList("hivemall.ftvec.pairing"));
- funcsHeaders.put("## Ranking", Collections.singletonList("hivemall.ftvec.ranking"));
- funcsHeaders.put("## Feature scaling", Collections.singletonList("hivemall.ftvec.scaling"));
- funcsHeaders.put("## Feature selection",
- Collections.singletonList("hivemall.ftvec.selection"));
- funcsHeaders.put("## Feature transformation and vectorization",
- Collections.singletonList("hivemall.ftvec.trans"));
- funcsHeaders.put("# Geospatial functions",
- Collections.singletonList("hivemall.geospatial"));
- funcsHeaders.put("# Distance measures", Collections.singletonList("hivemall.knn.distance"));
- funcsHeaders.put("# Locality-sensitive hashing",
- Collections.singletonList("hivemall.knn.lsh"));
- funcsHeaders.put("# Similarity measures",
- Collections.singletonList("hivemall.knn.similarity"));
- funcsHeaders.put("# Evaluation", Collections.singletonList("hivemall.evaluation"));
- funcsHeaders.put("# Sketching", Collections.singletonList("hivemall.sketch.hll"));
- funcsHeaders.put("# Ensemble learning", Collections.singletonList("hivemall.ensemble"));
- funcsHeaders.put("## Bagging", Collections.singletonList("hivemall.ensemble.bagging"));
- funcsHeaders.put("# Decision trees and RandomForest", Arrays.asList(
- "hivemall.smile.classification", "hivemall.smile.regression", "hivemall.smile.tools"));
- funcsHeaders.put("# XGBoost", Arrays.asList("hivemall.xgboost.classification",
- "hivemall.xgboost.regression", "hivemall.xgboost.tools"));
- funcsHeaders.put("# Others",
- Arrays.asList("hivemall", "hivemall.dataset", "hivemall.ftvec.text"));
- }
-
- @Override
- public void execute() throws MojoExecutionException {
- if (!isReactorRootProject()) {
- // output only once across the projects
- return;
- }
-
- generate(new File(basedir, pathToGenericFuncs),
- "This page describes a list of useful Hivemall generic functions. See also a [list of machine-learning-related functions](./funcs.md).",
- genericFuncsHeaders);
- generate(new File(basedir, pathToFuncs),
- "This page describes a list of Hivemall functions. See also a [list of generic Hivemall functions](./generic_funcs.md) for more general-purpose functions such as array and map UDFs.",
- funcsHeaders);
- }
-
- private boolean isReactorRootProject() {
- return session.getExecutionRootDirectory().equalsIgnoreCase(basedir.toString());
- }
-
- private void generate(@Nonnull File outputFile, @Nonnull String preface,
- @Nonnull Map<String, List<String>> headers) throws MojoExecutionException {
- Reflections reflections = new Reflections("hivemall");
- Set<Class<?>> annotatedClasses = reflections.getTypesAnnotatedWith(Description.class);
-
- StringBuilder sb = new StringBuilder();
- Map<String, Set<String>> packages = new HashMap<>();
-
- Pattern func = Pattern.compile("_FUNC_(\\(.*?\\))(.*)", Pattern.DOTALL);
-
- for (Class<?> annotatedClass : annotatedClasses) {
- Deprecated deprecated = annotatedClass.getAnnotation(Deprecated.class);
- if (deprecated != null) {
- continue;
- }
-
- Description description = annotatedClass.getAnnotation(Description.class);
-
- String value = description.value().replaceAll("\n", " ");
- Matcher matcher = func.matcher(value);
- if (matcher.find()) {
- value = asInlineCode(description.name() + matcher.group(1))
- + escapeHtml(matcher.group(2));
- }
- sb.append(asListElement(value));
-
- StringBuilder sbExtended = new StringBuilder();
- if (!description.extended().isEmpty()) {
- sbExtended.append(description.extended());
- sb.append("\n");
- }
-
- String extended = sbExtended.toString();
- if (extended.isEmpty()) {
- sb.append("\n");
- } else {
- if (extended.toLowerCase().contains("select")) { // extended description contains SQL statements
- sb.append(indent(asCodeBlock(extended, "sql")));
- } else {
- sb.append(indent(asCodeBlock(extended)));
- }
- }
-
- String packageName = annotatedClass.getPackage().getName();
- if (!packages.containsKey(packageName)) {
- Set<String> set = new TreeSet<>();
- packages.put(packageName, set);
- }
- Set<String> List = packages.get(packageName);
- List.add(sb.toString());
-
- StringUtils.clear(sb);
- }
-
- try (PrintWriter writer = new PrintWriter(outputFile)) {
- // license header
- writer.println("<!--");
- try {
- File licenseFile = new File(basedir, "resources/license-header.txt");
- FileReader fileReader = new FileReader(licenseFile);
-
- try (BufferedReader bufferedReader = new BufferedReader(fileReader)) {
- String line;
- while ((line = bufferedReader.readLine()) != null) {
- writer.println(indent(line));
- }
- }
- } catch (IOException e) {
- throw new MojoExecutionException("Failed to read license file");
- }
- writer.println("-->\n");
-
- writer.println(preface);
-
- writer.println("\n<!-- toc -->\n");
-
- for (Map.Entry<String, List<String>> e : headers.entrySet()) {
- writer.println(e.getKey() + "\n");
- List<String> packageNames = e.getValue();
- for (String packageName : packageNames) {
- for (String desc : packages.get(packageName)) {
- writer.println(desc);
- }
- }
- }
-
- writer.flush();
- } catch (FileNotFoundException e) {
- throw new MojoExecutionException("Output file is not found");
- }
- }
-}
[4/4] incubator-hivemall git commit: [HIVEMALL-145] Merge Brickhouse
functions
Posted by my...@apache.org.
[HIVEMALL-145] Merge Brickhouse functions
## What changes were proposed in this pull request?
Merge [brickhouse](https://github.com/klout/brickhouse) functions.
## What type of PR is it?
Feature
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-145
## How was this patch tested?
unit tests and manual tests
## How to use this feature?
as described in [user guide](http://hivemall.incubator.apache.org/userguide/misc/generic_funcs.html).
## Checklist
- [x] Did you apply source code formatter, i.e., `mvn formatter:format`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?
- [x] Invite active/main Brickhouse developers as Hivemall PPMC members or committers.
https://github.com/klout/brickhouse/issues/149
- [x] +1 from Klout members to merge
Author: Makoto Yui <my...@apache.org>
Closes #135 from myui/merge_brickhouse.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/49496032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/49496032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/49496032
Branch: refs/heads/master
Commit: 49496032498182baa60e5f93a1f3717359909433
Parents: 7ea6bfd
Author: Makoto Yui <my...@apache.org>
Authored: Wed Jun 6 18:09:17 2018 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Wed Jun 6 18:09:17 2018 +0900
----------------------------------------------------------------------
NOTICE | 2 +-
bin/update_func_md.sh | 47 ++
.../main/java/hivemall/HivemallVersionUDF.java | 2 +-
.../java/hivemall/sketch/bloom/BloomAndUDF.java | 3 +-
.../sketch/bloom/BloomContainsAnyUDF.java | 111 ++++
.../hivemall/sketch/bloom/BloomContainsUDF.java | 85 ++-
.../hivemall/sketch/bloom/BloomFilterUDAF.java | 114 ++++
.../java/hivemall/sketch/bloom/BloomNotUDF.java | 3 +-
.../java/hivemall/sketch/bloom/BloomOrUDF.java | 3 +-
.../java/hivemall/sketch/bloom/BloomUDAF.java | 101 ----
.../smile/classification/DecisionTree.java | 6 +-
.../smile/regression/RegressionTree.java | 10 +-
.../hivemall/smile/tools/TreePredictUDF.java | 2 +-
.../hivemall/statistics/MovingAverageUDTF.java | 84 ---
.../java/hivemall/tools/GenerateSeriesUDTF.java | 190 +++++--
.../main/java/hivemall/tools/TryCastUDF.java | 4 +-
.../hivemall/tools/array/ArrayAppendUDF.java | 11 +-
.../hivemall/tools/array/ArrayConcatUDF.java | 6 +-
.../hivemall/tools/array/ArrayElementAtUDF.java | 4 +-
.../hivemall/tools/array/ArrayFlattenUDF.java | 39 +-
.../hivemall/tools/array/ArrayIntersectUDF.java | 7 +-
.../hivemall/tools/array/ArrayRemoveUDF.java | 4 +-
.../hivemall/tools/array/ArraySliceUDF.java | 41 +-
.../hivemall/tools/array/ArrayToStrUDF.java | 96 ++++
.../hivemall/tools/array/ArrayUnionUDF.java | 8 +-
.../tools/array/ConditionalEmitUDTF.java | 19 +-
.../hivemall/tools/array/FirstElementUDF.java | 4 +-
.../hivemall/tools/array/LastElementUDF.java | 3 +-
.../tools/array/SortAndUniqArrayUDF.java | 2 +-
.../tools/array/SubarrayEndWithUDF.java | 2 +-
.../tools/array/SubarrayStartWithUDF.java | 2 +-
.../java/hivemall/tools/bits/BitsORUDF.java | 4 +-
.../java/hivemall/tools/bits/ToBitsUDF.java | 2 +-
.../java/hivemall/tools/bits/UnBitsUDF.java | 2 +-
.../hivemall/tools/compress/DeflateUDF.java | 2 +-
.../hivemall/tools/compress/InflateUDF.java | 4 +-
.../hivemall/tools/datetime/SessionizeUDF.java | 99 ++++
.../java/hivemall/tools/json/FromJsonUDF.java | 40 +-
.../java/hivemall/tools/json/ToJsonUDF.java | 76 ++-
.../hivemall/tools/list/UDAFToOrderedList.java | 16 +-
.../hivemall/tools/map/MapExcludeKeysUDF.java | 100 ++++
.../hivemall/tools/map/MapIncludeKeysUDF.java | 103 ++++
.../java/hivemall/tools/map/MapIndexUDF.java | 104 ++++
.../hivemall/tools/map/MapKeyValuesUDF.java | 97 ++++
.../java/hivemall/tools/map/MergeMapsUDAF.java | 172 ++++++
.../hivemall/tools/mapred/RowNumberUDF.java | 4 +-
.../java/hivemall/tools/sanity/AssertUDF.java | 6 +-
.../hivemall/tools/sanity/RaiseErrorUDF.java | 43 +-
.../java/hivemall/tools/text/Base91UDF.java | 2 +-
.../tools/text/NormalizeUnicodeUDF.java | 4 +-
.../hivemall/tools/text/SingularizeUDF.java | 2 +-
.../java/hivemall/tools/text/SplitWordsUDF.java | 2 +-
.../java/hivemall/tools/text/Unbase91UDF.java | 4 +-
.../java/hivemall/tools/text/WordNgramsUDF.java | 4 +-
.../tools/timeseries/MovingAverageUDTF.java | 98 ++++
.../hivemall/tools/vector/VectorAddUDF.java | 3 +-
.../hivemall/tools/vector/VectorDotUDF.java | 21 +-
.../java/hivemall/utils/hadoop/HiveUtils.java | 2 +-
.../java/hivemall/utils/math/MatrixUtils.java | 7 +-
.../FieldAwareFactorizationMachineUDTFTest.java | 2 +
.../ftvec/hashing/FeatureHashingUDFTest.java | 6 +-
.../smile/classification/DecisionTreeTest.java | 10 +-
.../smile/regression/RegressionTreeTest.java | 5 +-
.../statistics/MovingAverageUDTFTest.java | 78 ---
.../hivemall/tools/GenerateSeriesUDTFTest.java | 246 +++++++++
.../tools/array/ArrayAppendUDFTest.java | 9 +-
.../tools/array/ArrayFlattenUDFTest.java | 4 +-
.../hivemall/tools/array/ArraySliceUDFTest.java | 4 +-
.../hivemall/tools/array/ArrayToStrUDFTest.java | 103 ++++
.../hivemall/tools/array/ArrayUnionUDFTest.java | 1 +
.../tools/array/ConditionalEmitUDTFTest.java | 3 +-
.../tools/datetime/SessionizeUDFTest.java | 76 +++
.../hivemall/tools/json/FromJsonUDFTest.java | 1 +
.../java/hivemall/tools/json/ToJsonUDFTest.java | 6 +-
.../hivemall/tools/map/MapKeyValuesUDFTest.java | 81 +++
.../tools/sanity/RaiseErrorUDFTest.java | 10 +-
.../tools/timeseries/MovingAverageUDTFTest.java | 80 +++
.../hivemall/tools/vector/VectorAddUDFTest.java | 1 +
.../hivemall/tools/vector/VectorDotUDFTest.java | 7 +-
docs/gitbook/misc/funcs.md | 80 ++-
docs/gitbook/misc/generic_funcs.md | 531 +++++++++++++++----
pom.xml | 12 +
resources/ddl/define-all-as-permanent.hive | 99 +++-
resources/ddl/define-all.hive | 99 +++-
resources/ddl/define-all.spark | 95 +++-
resources/ddl/define-udfs.td.hql | 30 +-
spark/spark-2.2/pom.xml | 2 +-
tools/hivemall-docs/pom.xml | 113 +++-
.../java/hivemall/docs/FuncsListGenerator.java | 249 ---------
.../hivemall/docs/FuncsListGeneratorMojo.java | 264 +++++++++
...rojectDependenciesComponentConfigurator.java | 100 ++++
91 files changed, 3621 insertions(+), 824 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index 385a198..b4e5ace 100644
--- a/NOTICE
+++ b/NOTICE
@@ -10,4 +10,4 @@ the following organizations and individuals:
- Copyright 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
- Copyright 2015-2016 Makoto Yui
- Copyright 2015-2016 Treasure Data, Inc.
- - Copyright 2012 Klout, Inc.
\ No newline at end of file
+ - Copyright 2012 Klout, Inc.
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/bin/update_func_md.sh
----------------------------------------------------------------------
diff --git a/bin/update_func_md.sh b/bin/update_func_md.sh
new file mode 100755
index 0000000..bb0afda
--- /dev/null
+++ b/bin/update_func_md.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ "$HIVEMALL_HOME" = "" ]; then
+ if [ -e ../bin/${0##*/} ]; then
+ HIVEMALL_HOME=".."
+ elif [ -e ./bin/${0##*/} ]; then
+ HIVEMALL_HOME="."
+ else
+ echo "env HIVEMALL_HOME not defined"
+ exit 1
+ fi
+fi
+
+cd $HIVEMALL_HOME
+HIVEMALL_HOME=`pwd`
+
+# Deploy to local Maven repos
+
+export MAVEN_OPTS=-XX:MaxPermSize=256m
+mvn clean install -DskipTests=true -Dmaven.test.skip=true -pl '.,core,nlp,xgboost,tools/hivemall-docs'
+
+# Generate docs
+
+mvn org.apache.hivemall:hivemall-docs:generate-funcs-list -pl '.,core,nlp,xgboost,tools/hivemall-docs' -X
+
+# Run HTTP server on localhost:040
+
+cd ${HIVEMALL_HOME}/docs/gitbook
+gitbook install && gitbook serve
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/HivemallVersionUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/HivemallVersionUDF.java b/core/src/main/java/hivemall/HivemallVersionUDF.java
index c42701f..92636d5 100644
--- a/core/src/main/java/hivemall/HivemallVersionUDF.java
+++ b/core/src/main/java/hivemall/HivemallVersionUDF.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.io.Text;
@Description(name = "hivemall_version", value = "_FUNC_() - Returns the version of Hivemall",
- extended = "Usage: SELECT hivemall_version();")
+ extended = "SELECT hivemall_version();")
@UDFType(deterministic = true, stateful = false)
public final class HivemallVersionUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomAndUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomAndUDF.java b/core/src/main/java/hivemall/sketch/bloom/BloomAndUDF.java
index 87769da..f723acd 100644
--- a/core/src/main/java/hivemall/sketch/bloom/BloomAndUDF.java
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomAndUDF.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.util.bloom.DynamicBloomFilter;
import org.apache.hadoop.util.bloom.Filter;
@Description(name = "bloom_and",
- value = "_FUNC_(string bloom1, string bloom2) - Returns the logical AND of two bloom filters")
+ value = "_FUNC_(string bloom1, string bloom2) - Returns the logical AND of two bloom filters",
+ extended = "SELECT bloom_and(bf1, bf2) FROM xxx;")
@UDFType(deterministic = true, stateful = false)
public final class BloomAndUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomContainsAnyUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomContainsAnyUDF.java b/core/src/main/java/hivemall/sketch/bloom/BloomContainsAnyUDF.java
new file mode 100644
index 0000000..d8ff28b
--- /dev/null
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomContainsAnyUDF.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.sketch.bloom;
+
+import java.io.IOException;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.bloom.DynamicBloomFilter;
+import org.apache.hadoop.util.bloom.Filter;
+import org.apache.hadoop.util.bloom.Key;
+
+//@formatter:off
+@Description(name = "bloom_contains_any",
+ value = "_FUNC_(string bloom, string key) or _FUNC_(string bloom, array<string> keys)"
+ + "- Returns true if the bloom filter contains any of the given key",
+ extended = "WITH data1 as (\n" +
+ " SELECT explode(array(1,2,3,4,5)) as id\n" +
+ "),\n" +
+ "data2 as (\n" +
+ " SELECT explode(array(1,3,5,6,8)) as id\n" +
+ "),\n" +
+ "bloom as (\n" +
+ " SELECT bloom(id) as bf\n" +
+ " FROM data1\n" +
+ ")\n" +
+ "SELECT \n" +
+ " l.* \n" +
+ "FROM \n" +
+ " data2 l\n" +
+ " CROSS JOIN bloom r\n" +
+ "WHERE\n" +
+ " bloom_contains_any(r.bf, array(l.id))")
+//@formatter:on
+@UDFType(deterministic = true, stateful = false)
+public final class BloomContainsAnyUDF extends UDF {
+
+ @Nonnull
+ private final Key key = new Key();
+
+ @Nullable
+ private Text prevBfStr;
+ @Nullable
+ private Filter prevBf;
+
+ @Nullable
+ public Boolean evaluate(@Nullable Text bloomStr, @Nullable List<Text> keys)
+ throws HiveException {
+ if (bloomStr == null) {
+ return null;
+ }
+ if (keys == null) {
+ return Boolean.FALSE;
+ }
+
+ final Filter bloom = getFilter(bloomStr);
+
+ for (Text keyStr : keys) {
+ if (keyStr == null) {
+ continue;
+ }
+ key.set(keyStr.copyBytes(), 1.0d);
+ if (bloom.membershipTest(key)) {
+ return Boolean.TRUE;
+ }
+ }
+
+ return Boolean.FALSE;
+ }
+
+ @Nonnull
+ private Filter getFilter(@Nonnull final Text bloomStr) throws HiveException {
+ final Filter bloom;
+ if (prevBf != null && prevBfStr.equals(bloomStr)) {
+ bloom = prevBf;
+ } else {
+ try {
+ bloom = BloomFilterUtils.deserialize(bloomStr, new DynamicBloomFilter());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ this.prevBfStr = new Text(bloomStr);
+ this.prevBf = bloom;
+ }
+ return bloom;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomContainsUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomContainsUDF.java b/core/src/main/java/hivemall/sketch/bloom/BloomContainsUDF.java
index 2da65b3..f76b85d 100644
--- a/core/src/main/java/hivemall/sketch/bloom/BloomContainsUDF.java
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomContainsUDF.java
@@ -19,6 +19,7 @@
package hivemall.sketch.bloom;
import java.io.IOException;
+import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -32,8 +33,37 @@ import org.apache.hadoop.util.bloom.DynamicBloomFilter;
import org.apache.hadoop.util.bloom.Filter;
import org.apache.hadoop.util.bloom.Key;
+//@formatter:off
@Description(name = "bloom_contains",
- value = "_FUNC_(string bloom, string key) - Returns true if the bloom filter contains the given key")
+ value = "_FUNC_(string bloom, string key) or _FUNC_(string bloom, array<string> keys)"
+ + " - Returns true if the bloom filter contains all the given key(s). Returns false if key is null.",
+ extended = "WITH satisfied_movies as (\n" +
+ " SELECT bloom(movieid) as movies\n" +
+ " FROM (\n" +
+ " SELECT movieid\n" +
+ " FROM ratings\n" +
+ " GROUP BY movieid\n" +
+ " HAVING avg(rating) >= 4.0\n" +
+ " ) t\n" +
+ ")\n" +
+ "SELECT\n" +
+ " l.rating,\n" +
+ " count(distinct l.userid) as cnt\n" +
+ "FROM\n" +
+ " ratings l \n" +
+ " CROSS JOIN satisfied_movies r\n" +
+ "WHERE\n" +
+ " bloom_contains(r.movies, l.movieid) -- includes false positive\n" +
+ "GROUP BY \n" +
+ " l.rating;\n" +
+ "\n" +
+ "l.rating cnt\n" +
+ "1 1296\n" +
+ "2 2770\n" +
+ "3 5008\n" +
+ "4 5824\n" +
+ "5 5925")
+//@formatter:on
@UDFType(deterministic = true, stateful = false)
public final class BloomContainsUDF extends UDF {
@@ -41,31 +71,64 @@ public final class BloomContainsUDF extends UDF {
private final Key key = new Key();
@Nullable
- private Text prevKey;
+ private Text prevBfStr;
@Nullable
- private Filter prevFilter;
+ private Filter prevBf;
@Nullable
public Boolean evaluate(@Nullable Text bloomStr, @Nullable Text keyStr) throws HiveException {
- if (bloomStr == null || key == null) {
+ if (bloomStr == null) {
return null;
}
+ if (keyStr == null) {
+ return Boolean.FALSE;
+ }
+
+ Filter bloom = getFilter(bloomStr);
+ key.set(keyStr.copyBytes(), 1.0d);
+ return Boolean.valueOf(bloom.membershipTest(key));
+ }
+
+ @Nullable
+ public Boolean evaluate(@Nullable Text bloomStr, @Nullable List<Text> keys)
+ throws HiveException {
+ if (bloomStr == null) {
+ return null;
+ }
+ if (keys == null) {
+ return Boolean.FALSE;
+ }
+
+ final Filter bloom = getFilter(bloomStr);
+ for (Text keyStr : keys) {
+ if (keyStr == null) {
+ continue;
+ }
+ key.set(keyStr.copyBytes(), 1.0d);
+ if (bloom.membershipTest(key) == false) {
+ return Boolean.FALSE;
+ }
+ }
+
+ return Boolean.TRUE;
+ }
+
+ @Nonnull
+ private Filter getFilter(@Nonnull final Text bloomStr) throws HiveException {
final Filter bloom;
- if (prevFilter != null && prevKey.equals(keyStr)) {
- bloom = prevFilter;
+ if (prevBf != null && prevBfStr.equals(bloomStr)) {
+ bloom = prevBf;
} else {
try {
bloom = BloomFilterUtils.deserialize(bloomStr, new DynamicBloomFilter());
} catch (IOException e) {
throw new HiveException(e);
}
- this.prevKey = keyStr;
- this.prevFilter = bloom;
- key.set(keyStr.getBytes(), 1.0d);
+ this.prevBfStr = new Text(bloomStr);
+ this.prevBf = bloom;
}
-
- return Boolean.valueOf(bloom.membershipTest(key));
+ return bloom;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomFilterUDAF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomFilterUDAF.java b/core/src/main/java/hivemall/sketch/bloom/BloomFilterUDAF.java
new file mode 100644
index 0000000..a76a60c
--- /dev/null
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomFilterUDAF.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.sketch.bloom;
+
+import java.io.IOException;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.bloom.DynamicBloomFilter;
+import org.apache.hadoop.util.bloom.Filter;
+import org.apache.hadoop.util.bloom.Key;
+
+//@formatter:off
+@Description(name = "bloom",
+ value = "_FUNC_(string key) - Constructs a BloomFilter by aggregating a set of keys",
+ extended = "CREATE TABLE satisfied_movies AS \n" +
+ " SELECT bloom(movieid) as movies\n" +
+ " FROM (\n" +
+ " SELECT movieid\n" +
+ " FROM ratings\n" +
+ " GROUP BY movieid\n" +
+ " HAVING avg(rating) >= 4.0\n" +
+ " ) t;")
+//@formatter:on
+@SuppressWarnings("deprecation")
+public final class BloomFilterUDAF extends UDAF {
+
+ public static class Evaluator implements UDAFEvaluator {
+
+ private Filter filter;
+ private Key key;
+
+ @Override
+ public void init() {
+ this.filter = BloomFilterUtils.newDynamicBloomFilter();
+ this.key = new Key();
+ }
+
+ public boolean iterate(@Nullable Text keyStr) {
+ if (keyStr == null) {
+ return true;
+ }
+ if (filter == null) {
+ init();
+ }
+
+ key.set(keyStr.copyBytes(), 1.0d);
+ filter.add(key);
+
+ return true;
+ }
+
+ @Nonnull
+ public Text terminatePartial() throws HiveException {
+ try {
+ return BloomFilterUtils.serialize(filter, new Text());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ public boolean merge(@Nonnull Text partial) throws HiveException {
+ final DynamicBloomFilter other;
+ try {
+ other = BloomFilterUtils.deserialize(partial, new DynamicBloomFilter());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+
+ if (filter == null) {
+ this.filter = other;
+ } else {
+ filter.or(other);
+ }
+ return true;
+ }
+
+ @Nullable
+ public Text terminate() throws HiveException {
+ if (filter == null) {
+ return null;
+ }
+
+ try {
+ return BloomFilterUtils.serialize(filter, new Text());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomNotUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomNotUDF.java b/core/src/main/java/hivemall/sketch/bloom/BloomNotUDF.java
index cd385e3..1a074b0 100644
--- a/core/src/main/java/hivemall/sketch/bloom/BloomNotUDF.java
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomNotUDF.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.util.bloom.DynamicBloomFilter;
import org.apache.hadoop.util.bloom.Filter;
@Description(name = "bloom_not",
- value = "_FUNC_(string bloom) - Returns the logical NOT of a bloom filters")
+ value = "_FUNC_(string bloom) - Returns the logical NOT of a bloom filters",
+ extended = "SELECT bloom_not(bf) FROM xxx;")
@UDFType(deterministic = true, stateful = false)
public final class BloomNotUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomOrUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomOrUDF.java b/core/src/main/java/hivemall/sketch/bloom/BloomOrUDF.java
index 7d2980e..0fcb539 100644
--- a/core/src/main/java/hivemall/sketch/bloom/BloomOrUDF.java
+++ b/core/src/main/java/hivemall/sketch/bloom/BloomOrUDF.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.util.bloom.DynamicBloomFilter;
import org.apache.hadoop.util.bloom.Filter;
@Description(name = "bloom_or",
- value = "_FUNC_(string bloom1, string bloom2) - Returns the logical OR of two bloom filters")
+ value = "_FUNC_(string bloom1, string bloom2) - Returns the logical OR of two bloom filters",
+ extended = "SELECT bloom_or(bf1, bf2) FROM xxx;")
@UDFType(deterministic = true, stateful = false)
public final class BloomOrUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/sketch/bloom/BloomUDAF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/sketch/bloom/BloomUDAF.java b/core/src/main/java/hivemall/sketch/bloom/BloomUDAF.java
deleted file mode 100644
index cb09b93..0000000
--- a/core/src/main/java/hivemall/sketch/bloom/BloomUDAF.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.sketch.bloom;
-
-import java.io.IOException;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDAF;
-import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.bloom.DynamicBloomFilter;
-import org.apache.hadoop.util.bloom.Filter;
-import org.apache.hadoop.util.bloom.Key;
-
-@Description(name = "bloom",
- value = "_FUNC_(string key) - Constructs a BloomFilter by aggregating a set of keys")
-@SuppressWarnings("deprecation")
-public final class BloomUDAF extends UDAF {
-
- public static class Evaluator implements UDAFEvaluator {
-
- private Filter filter;
- private Key key;
-
- @Override
- public void init() {
- this.filter = BloomFilterUtils.newDynamicBloomFilter();
- this.key = new Key();
- }
-
- public boolean iterate(@Nullable Text keyStr) {
- if (keyStr == null) {
- return true;
- }
- key.set(keyStr.getBytes(), 1.0d);
-
- filter.add(key);
-
- return true;
- }
-
- @Nonnull
- public Text terminatePartial() throws HiveException {
- try {
- return BloomFilterUtils.serialize(filter, new Text());
- } catch (IOException e) {
- throw new HiveException(e);
- }
- }
-
- public boolean merge(@Nonnull Text partial) throws HiveException {
- final DynamicBloomFilter other;
- try {
- other = BloomFilterUtils.deserialize(partial, new DynamicBloomFilter());
- } catch (IOException e) {
- throw new HiveException(e);
- }
-
- if (filter == null) {
- this.filter = other;
- } else {
- filter.or(other);
- }
- return true;
- }
-
- @Nullable
- public Text terminate() throws HiveException {
- if (filter == null) {
- return null;
- }
-
- try {
- return BloomFilterUtils.serialize(filter, new Text());
- } catch (IOException e) {
- throw new HiveException(e);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/smile/classification/DecisionTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/smile/classification/DecisionTree.java b/core/src/main/java/hivemall/smile/classification/DecisionTree.java
index cc92ae8..a80a299 100644
--- a/core/src/main/java/hivemall/smile/classification/DecisionTree.java
+++ b/core/src/main/java/hivemall/smile/classification/DecisionTree.java
@@ -19,6 +19,7 @@ package hivemall.smile.classification;
import static hivemall.smile.utils.SmileExtUtils.resolveFeatureName;
import static hivemall.smile.utils.SmileExtUtils.resolveName;
+
import hivemall.annotations.VisibleForTesting;
import hivemall.math.matrix.Matrix;
import hivemall.math.matrix.ints.ColumnMajorIntMatrix;
@@ -36,6 +37,8 @@ import hivemall.utils.lang.ObjectUtils;
import hivemall.utils.lang.StringUtils;
import hivemall.utils.lang.mutable.MutableInt;
import hivemall.utils.sampling.IntReservoirSampler;
+import smile.classification.Classifier;
+import smile.math.Math;
import java.io.Externalizable;
import java.io.IOException;
@@ -53,9 +56,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.roaringbitmap.IntConsumer;
import org.roaringbitmap.RoaringBitmap;
-import smile.classification.Classifier;
-import smile.math.Math;
-
/**
* Decision tree for classification. A decision tree can be learned by splitting the training set
* into subsets based on an attribute value test. This process is repeated on each derived subset in
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/smile/regression/RegressionTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/smile/regression/RegressionTree.java b/core/src/main/java/hivemall/smile/regression/RegressionTree.java
index b8a3cc7..61964ae 100755
--- a/core/src/main/java/hivemall/smile/regression/RegressionTree.java
+++ b/core/src/main/java/hivemall/smile/regression/RegressionTree.java
@@ -18,6 +18,7 @@
package hivemall.smile.regression;
import static hivemall.smile.utils.SmileExtUtils.resolveFeatureName;
+
import hivemall.annotations.VisibleForTesting;
import hivemall.math.matrix.Matrix;
import hivemall.math.matrix.ints.ColumnMajorIntMatrix;
@@ -36,6 +37,10 @@ import hivemall.utils.lang.ObjectUtils;
import hivemall.utils.lang.StringUtils;
import hivemall.utils.lang.mutable.MutableInt;
import hivemall.utils.math.MathUtils;
+import smile.math.Math;
+import smile.regression.GradientTreeBoost;
+import smile.regression.RandomForest;
+import smile.regression.Regression;
import java.io.Externalizable;
import java.io.IOException;
@@ -51,11 +56,6 @@ import javax.annotation.Nullable;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import smile.math.Math;
-import smile.regression.GradientTreeBoost;
-import smile.regression.RandomForest;
-import smile.regression.Regression;
-
/**
* Decision tree for regression. A decision tree can be learned by splitting the training set into
* subsets based on an attribute value test. This process is repeated on each derived subset in a
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/smile/tools/TreePredictUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/smile/tools/TreePredictUDF.java b/core/src/main/java/hivemall/smile/tools/TreePredictUDF.java
index 9b775bf..511944c 100644
--- a/core/src/main/java/hivemall/smile/tools/TreePredictUDF.java
+++ b/core/src/main/java/hivemall/smile/tools/TreePredictUDF.java
@@ -158,7 +158,7 @@ public final class TreePredictUDF extends UDFWithOptions {
Object arg2 = arguments[2].get();
if (arg2 == null) {
- throw new HiveException("array<double> features was null");
+ throw new HiveException("features was null");
}
this.featuresProbe = parseFeatures(arg2, featuresProbe);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/statistics/MovingAverageUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/statistics/MovingAverageUDTF.java b/core/src/main/java/hivemall/statistics/MovingAverageUDTF.java
deleted file mode 100644
index 112c47f..0000000
--- a/core/src/main/java/hivemall/statistics/MovingAverageUDTF.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.statistics;
-
-import hivemall.utils.hadoop.HiveUtils;
-import hivemall.utils.stats.MovingAverage;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-
-@Description(name = "moving_avg", value = "_FUNC_(NUMBER value, const int windowSize)"
- + " - Returns moving average of a time series using a given window")
-public final class MovingAverageUDTF extends GenericUDTF {
-
- private PrimitiveObjectInspector valueOI;
-
- private MovingAverage movingAvg;
-
- private Object[] forwardObjs;
- private DoubleWritable result;
-
- @Override
- public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
- if (argOIs.length != 2) {
- throw new UDFArgumentException(
- "Two argument is expected for moving_avg(NUMBER value, const int windowSize): "
- + argOIs.length);
- }
- this.valueOI = HiveUtils.asNumberOI(argOIs[0]);
-
- int windowSize = HiveUtils.getConstInt(argOIs[1]);
- this.movingAvg = new MovingAverage(windowSize);
-
- this.result = new DoubleWritable();
- this.forwardObjs = new Object[] {result};
-
- List<String> fieldNames = Arrays.asList("avg");
- List<ObjectInspector> fieldOIs = Arrays.<ObjectInspector>asList(
- PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
-
- return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
- }
-
- @Override
- public void process(Object[] args) throws HiveException {
- double x = HiveUtils.getDouble(args[0], valueOI);
-
- double avg = movingAvg.add(x);
- result.set(avg);
-
- forward(forwardObjs);
- }
-
- @Override
- public void close() throws HiveException {}
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java b/core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java
index 2567ac7..ed9a7a2 100644
--- a/core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java
+++ b/core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java
@@ -21,6 +21,10 @@ package hivemall.tools;
import hivemall.utils.hadoop.HiveUtils;
import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -28,70 +32,176 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
+// @formatter:off
@Description(name = "generate_series",
value = "_FUNC_(const int|bigint start, const int|bigint end) - "
- + "Generate a series of values, from start to end. A similar function to PostgreSQL's `generate_serics`. http://www.postgresql.org/docs/current/static/functions-srf.html",
- extended = "select generate_series(1,9);\n\n" + "1\n" + "2\n" + "3\n" + "4\n" + "5\n"
- + "6\n" + "7\n" + "8\n" + "9")
+ + "Generate a series of values, from start to end. " +
+ "A similar function to PostgreSQL's [generate_serics](http://www.postgresql.org/docs/current/static/functions-srf.html)",
+ extended = "SELECT generate_series(2,4);\n" +
+ "\n" +
+ " 2\n" +
+ " 3\n" +
+ " 4\n" +
+ "\n" +
+ "SELECT generate_series(5,1,-2);\n" +
+ "\n" +
+ " 5\n" +
+ " 3\n" +
+ " 1\n" +
+ "\n" +
+ "SELECT generate_series(4,3);\n" +
+ "\n" +
+ " (no return)\n" +
+ "\n" +
+ "SELECT date_add(current_date(),value),value from (SELECT generate_series(1,3)) t;\n" +
+ "\n" +
+ " 2018-04-21 1\n" +
+ " 2018-04-22 2\n" +
+ " 2018-04-23 3\n" +
+ "\n" +
+ "WITH input as (\n" +
+ " SELECT 1 as c1, 10 as c2, 3 as step\n" +
+ " UNION ALL\n" +
+ " SELECT 10, 2, -3\n" +
+ ")\n" +
+ "SELECT generate_series(c1, c2, step) as series\n" +
+ "FROM input;\n" +
+ "\n" +
+ " 1\n" +
+ " 4\n" +
+ " 7\n" +
+ " 10\n" +
+ " 10\n" +
+ " 7\n" +
+ " 4")
+// @formatter:on
public final class GenerateSeriesUDTF extends GenericUDTF {
- private long start, end;
- private boolean useBigInt;
+ private PrimitiveObjectInspector startOI, endOI;
+ @Nullable
+ private PrimitiveObjectInspector stepOI;
+
+ @Nonnull
+ private final Writable[] row = new Writable[1];
+ private boolean returnLong;
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
- if (argOIs.length != 2) {
- throw new UDFArgumentException("Expected number of arguments is 2: " + argOIs.length);
+ if (argOIs.length != 2 && argOIs.length != 3) {
+ throw new UDFArgumentException(
+ "Expected number of arguments is 2 or 3: " + argOIs.length);
+ }
+ if (!HiveUtils.isIntegerOI(argOIs[0])) {
+ throw new UDFArgumentException(
+ "Expected Integer type for the first argument: " + argOIs[0].getTypeName());
+ }
+ if (!HiveUtils.isIntegerOI(argOIs[1])) {
+ throw new UDFArgumentException(
+ "Expected Integer type for the second argument: " + argOIs[1].getTypeName());
}
+ this.startOI = HiveUtils.asIntegerOI(argOIs[0]);
+ this.endOI = HiveUtils.asIntegerOI(argOIs[1]);
- ArrayList<String> fieldNames = new ArrayList<String>(1);
+ if (argOIs.length == 3) {
+ if (!HiveUtils.isIntegerOI(argOIs[2])) {
+ throw new UDFArgumentException(
+ "Expected Integer type for the third argument: " + argOIs[2].getTypeName());
+ }
+ this.stepOI = HiveUtils.asIntegerOI(argOIs[2]);
+ }
+
+ this.returnLong = HiveUtils.isBigIntOI(startOI) || HiveUtils.isBigIntOI(endOI);
+
+ List<String> fieldNames = new ArrayList<>(1);
fieldNames.add("value");
- ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(1);
+ List<ObjectInspector> fieldOIs = new ArrayList<>(1);
+ if (returnLong) {
+ fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ } else {
+ fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+ }
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
- this.useBigInt = HiveUtils.isBigIntOI(argOIs[1]);
- if (useBigInt) {
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
+ @Override
+ public void process(Object[] args) throws HiveException {
+ if (returnLong) {
+ generateLongSeries(args);
} else {
- fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ generateIntSeries(args);
}
+ }
- this.start = HiveUtils.getAsConstLong(argOIs[0]);
- this.end = HiveUtils.getAsConstLong(argOIs[1]);
- if (start > end) {
- throw new UDFArgumentException(
- "start '" + start + "' must be less than or equals to end '" + end + "'");
+ private void generateLongSeries(@Nonnull final Object[] args) throws HiveException {
+ final long start, end;
+ long step = 1L;
+ switch (args.length) {
+ case 3:
+ step = PrimitiveObjectInspectorUtils.getLong(args[2], stepOI);
+ if (step == 0) {
+ throw new UDFArgumentException("Step MUST NOT be zero");
+ }
+ // fall through
+ case 2:
+ start = PrimitiveObjectInspectorUtils.getLong(args[0], startOI);
+ end = PrimitiveObjectInspectorUtils.getLong(args[1], endOI);
+ break;
+ default:
+ throw new UDFArgumentException("Expected number of arguments: " + args.length);
}
- return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ final LongWritable row0 = new LongWritable();
+ row[0] = row0;
+ if (step > 0) {
+ for (long i = start; i <= end; i += step) {
+ row0.set(i);
+ forward(row);
+ }
+ } else {
+ for (long i = start; i >= end; i += step) {
+ row0.set(i);
+ forward(row);
+ }
+ }
}
- @Override
- public void process(Object[] argOIs) throws HiveException {
- final Object[] forwardObjs = new Object[1];
- if (useBigInt) {
- if (start == end) {
- forwardObjs[0] = start;
- forward(forwardObjs);
- } else {
- for (long i = start; i <= end; i++) {
- forwardObjs[0] = i;
- forward(forwardObjs);
+ private void generateIntSeries(@Nonnull final Object[] args) throws HiveException {
+ final int start, end;
+ int step = 1;
+ switch (args.length) {
+ case 3:
+ step = PrimitiveObjectInspectorUtils.getInt(args[2], stepOI);
+ if (step == 0) {
+ throw new UDFArgumentException("Step MUST NOT be zero");
}
+ // fall through
+ case 2:
+ start = PrimitiveObjectInspectorUtils.getInt(args[0], startOI);
+ end = PrimitiveObjectInspectorUtils.getInt(args[1], endOI);
+ break;
+ default:
+ throw new UDFArgumentException("Expected number of arguments: " + args.length);
+ }
+
+ final IntWritable row0 = new IntWritable();
+ row[0] = row0;
+ if (step > 0) {
+ for (int i = start; i <= end; i += step) {
+ row0.set(i);
+ forward(row);
}
} else {
- int starti = (int) start;
- int endi = (int) end;
- if (starti == endi) {
- forwardObjs[0] = starti;
- forward(forwardObjs);
- } else {
- for (int i = starti; i <= endi; i++) {
- forwardObjs[0] = i;
- forward(forwardObjs);
- }
+ for (int i = start; i >= end; i += step) {
+ row0.set(i);
+ forward(row);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/TryCastUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/TryCastUDF.java b/core/src/main/java/hivemall/tools/TryCastUDF.java
index a0f3257..adb8328 100644
--- a/core/src/main/java/hivemall/tools/TryCastUDF.java
+++ b/core/src/main/java/hivemall/tools/TryCastUDF.java
@@ -35,8 +35,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@Description(name = "try_cast",
value = "_FUNC_(ANY src, const string typeName)"
+ " - Explicitly cast a value as a type. Returns null if cast fails.",
- extended = "Usage: select try_cast(array(1.0,2.0,3.0), 'array<string>')\n"
- + " select try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')")
+ extended = "SELECT try_cast(array(1.0,2.0,3.0), 'array<string>')\n"
+ + "SELECT try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')")
@UDFType(deterministic = true, stateful = false)
public final class TryCastUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayAppendUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayAppendUDF.java b/core/src/main/java/hivemall/tools/array/ArrayAppendUDF.java
index 25d0f4c..b3768bb 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayAppendUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayAppendUDF.java
@@ -21,6 +21,7 @@ package hivemall.tools.array;
import hivemall.utils.hadoop.HiveUtils;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
@@ -38,7 +39,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@Description(name = "array_append",
- value = "_FUNC_(array<T> arr, T elem) - Append an element to the end of an array")
+ value = "_FUNC_(array<T> arr, T elem) - Append an element to the end of an array",
+ extended = "SELECT array_append(array(1,2),3);\n 1,2,3\n\n"
+ + "SELECT array_append(array('a','b'),'c');\n \"a\",\"b\",\"c\"")
@UDFType(deterministic = true, stateful = false)
public final class ArrayAppendUDF extends GenericUDF {
@@ -71,6 +74,12 @@ public final class ArrayAppendUDF extends GenericUDF {
Object arg0 = args[0].get();
if (arg0 == null) {
+ Object arg1 = args[1].get();
+ if (arg1 != null) {
+ Object toAppend = returnWritables ? primInspector.getPrimitiveWritableObject(arg1)
+ : primInspector.getPrimitiveJavaObject(arg1);
+ return Arrays.asList(toAppend);
+ }
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayConcatUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayConcatUDF.java b/core/src/main/java/hivemall/tools/array/ArrayConcatUDF.java
index 62e3e36..7781564 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayConcatUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayConcatUDF.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.Object
@Description(name = "array_concat",
value = "_FUNC_(array<ANY> x1, array<ANY> x2, ..) - Returns a concatenated array",
- extended = "select array_concat(array(1),array(2,3));\n" + "> [1,2,3]")
+ extended = "SELECT array_concat(array(1),array(2,3));\n" + " [1,2,3]")
@UDFType(deterministic = true, stateful = false)
public class ArrayConcatUDF extends GenericUDF {
/**
@@ -95,11 +95,11 @@ public class ArrayConcatUDF extends GenericUDF {
continue;
}
- final ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[i];
+ final ListObjectInspector arrayOI = argumentOIs[i];
+ final ObjectInspector elemOI = arrayOI.getListElementObjectInspector();
final int arraylength = arrayOI.getListLength(arrayObject);
for (int j = 0; j < arraylength; j++) {
Object rawObj = arrayOI.getListElement(arrayObject, j);
- ObjectInspector elemOI = arrayOI.getListElementObjectInspector();
Object obj = ObjectInspectorUtils.copyToStandardObject(rawObj, elemOI,
ObjectInspectorCopyOption.WRITABLE);
ret.add(obj);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayElementAtUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayElementAtUDF.java b/core/src/main/java/hivemall/tools/array/ArrayElementAtUDF.java
index 631d92d..21be8d4 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayElementAtUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayElementAtUDF.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
@Description(name = "element_at",
- value = "_FUNC_(array<T> list, int pos) - Returns an element at the given position")
+ value = "_FUNC_(array<T> list, int pos) - Returns an element at the given position",
+ extended = "SELECT element_at(array(1,2,3,4),0);\n 1\n\n"
+ + "SELECT element_at(array(1,2,3,4),-2);\n 3")
@UDFType(deterministic = true, stateful = false)
public final class ArrayElementAtUDF extends GenericUDF {
private ListObjectInspector listInspector;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayFlattenUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayFlattenUDF.java b/core/src/main/java/hivemall/tools/array/ArrayFlattenUDF.java
index 906d594..39d0110 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayFlattenUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayFlattenUDF.java
@@ -19,6 +19,7 @@
package hivemall.tools.array;
import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
@@ -33,16 +34,16 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@Description(name = "array_flatten",
- value = "_FUNC_(array<array<ANY>>) - Returns an array with the elements flattened.")
+ value = "_FUNC_(array<array<ANY>>) - Returns an array with the elements flattened.",
+ extended = "SELECT array_flatten(array(array(1,2,3),array(4,5),array(6,7,8)));\n"
+ + " [1,2,3,4,5,6,7,8]")
@UDFType(deterministic = true, stateful = false)
public final class ArrayFlattenUDF extends GenericUDF {
private ListObjectInspector listOI;
- private ListObjectInspector nextedListOI;
- private ObjectInspector elemOI;
-
private final List<Object> result = new ArrayList<>();
@Override
@@ -58,11 +59,13 @@ public final class ArrayFlattenUDF extends GenericUDF {
throw new UDFArgumentException(
"array_flatten takes array of array for the argument: " + listOI.toString());
}
- this.nextedListOI = HiveUtils.asListOI(listElemOI);
- this.elemOI = nextedListOI.getListElementObjectInspector();
+
+ ListObjectInspector nestedListOI = HiveUtils.asListOI(listElemOI);
+ ObjectInspector elemOI = nestedListOI.getListElementObjectInspector();
return ObjectInspectorFactory.getStandardListObjectInspector(
- ObjectInspectorUtils.getStandardObjectInspector(elemOI));
+ ObjectInspectorUtils.getStandardObjectInspector(elemOI,
+ ObjectInspectorCopyOption.WRITABLE));
}
@Override
@@ -81,12 +84,17 @@ public final class ArrayFlattenUDF extends GenericUDF {
continue;
}
- final int subarrayLength = nextedListOI.getListLength(subarray);
+ final ListObjectInspector subarrayOI =
+ HiveUtils.asListOI(listOI.getListElementObjectInspector());
+ final ObjectInspector elemOI = subarrayOI.getListElementObjectInspector();
+ final int subarrayLength = subarrayOI.getListLength(subarray);
for (int j = 0; j < subarrayLength; j++) {
- Object elem = nextedListOI.getListElement(subarray, j);
- if (elem == null) {
+ Object rawElem = subarrayOI.getListElement(subarray, j);
+ if (rawElem == null) {
continue;
}
+ Object elem = ObjectInspectorUtils.copyToStandardObject(rawElem, elemOI,
+ ObjectInspectorCopyOption.WRITABLE);
result.add(elem);
}
}
@@ -96,16 +104,7 @@ public final class ArrayFlattenUDF extends GenericUDF {
@Override
public String getDisplayString(String[] args) {
- final StringBuffer buf = new StringBuffer();
- buf.append("array_flatten(");
- for (int i = 0, len = args.length; i < len; i++) {
- if (i != 0) {
- buf.append(", ");
- }
- buf.append(args[i]);
- }
- buf.append(")");
- return buf.toString();
+ return "array_flatten(" + StringUtils.join(args, ',') + ")";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayIntersectUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayIntersectUDF.java b/core/src/main/java/hivemall/tools/array/ArrayIntersectUDF.java
index 909176a..4b7f91c 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayIntersectUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayIntersectUDF.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@Description(name = "array_intersect",
value = "_FUNC_(array<ANY> x1, array<ANY> x2, ..) - Returns an intersect of given arrays",
- extended = "select array_intersect(array(1,3,4),array(2,3,4),array(3,5));\n" + "> [3]")
+ extended = "SELECT array_intersect(array(1,3,4),array(2,3,4),array(3,5));\n" + " [3]")
@UDFType(deterministic = true, stateful = false)
public final class ArrayIntersectUDF extends GenericUDF {
@@ -88,7 +88,7 @@ public final class ArrayIntersectUDF extends GenericUDF {
return Collections.emptyList();
}
- Set<InspectableObject> checkSet = new HashSet<ArrayIntersectUDF.InspectableObject>();
+ Set<InspectableObject> checkSet = new HashSet<InspectableObject>();
final ListObjectInspector arg0ListOI = argListOIs[0];
final ObjectInspector arg0ElemOI = arg0ListOI.getListElementObjectInspector();
final int arg0size = arg0ListOI.getListLength(arg0);
@@ -106,8 +106,7 @@ public final class ArrayIntersectUDF extends GenericUDF {
if (argI == null) {
continue;
}
- final Set<InspectableObject> newSet =
- new HashSet<ArrayIntersectUDF.InspectableObject>();
+ final Set<InspectableObject> newSet = new HashSet<InspectableObject>();
final ListObjectInspector argIListOI = argListOIs[i];
final ObjectInspector argIElemOI = argIListOI.getListElementObjectInspector();
for (int j = 0, j_size = argIListOI.getListLength(argI); j < j_size; j++) {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayRemoveUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayRemoveUDF.java b/core/src/main/java/hivemall/tools/array/ArrayRemoveUDF.java
index 523093b..207c398 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayRemoveUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayRemoveUDF.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.io.Text;
@Description(name = "array_remove",
value = "_FUNC_(array<int|text> original, int|text|array<int> target)"
+ " - Returns an array that the target is removed " + "from the original array",
- extended = "select array_remove(array(1,null,3),array(null));\n" + "> [3]\n" + "\n"
- + "select array_remove(array(\"aaa\",\"bbb\"),\"bbb\");\n" + "> [\"aaa\"]")
+ extended = "SELECT array_remove(array(1,null,3),array(null));\n" + " [3]\n" + "\n"
+ + "SELECT array_remove(array(\"aaa\",\"bbb\"),\"bbb\");\n" + " [\"aaa\"]")
@UDFType(deterministic = true, stateful = false)
public class ArrayRemoveUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArraySliceUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArraySliceUDF.java b/core/src/main/java/hivemall/tools/array/ArraySliceUDF.java
index 2bc98b9..e842df6 100644
--- a/core/src/main/java/hivemall/tools/array/ArraySliceUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArraySliceUDF.java
@@ -39,9 +39,48 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+// @formatter:off
@Description(name = "array_slice",
value = "_FUNC_(array<ANY> values, int offset [, int length]) - Slices the given array by the given offset and length parameters.",
- extended = "select array_slice(array(1,2,3,4,5,6), 2,4);\n" + "> [3,4]")
+ extended = "SELECT \n" +
+ " array_slice(array(1,2,3,4,5,6), 2,4),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " 0, -- offset\n" +
+ " 2 -- length\n" +
+ " ),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " 6, -- offset\n" +
+ " 3 -- length\n" +
+ " ),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " 6, -- offset\n" +
+ " 10 -- length\n" +
+ " ),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " 6 -- offset\n" +
+ " ),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " -3 -- offset\n" +
+ " ),\n" +
+ " array_slice(\n" +
+ " array(\"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\", \"ten\"),\n" +
+ " -3, -- offset\n" +
+ " 2 -- length\n" +
+ " );\n" +
+ "\n" +
+ " [3,4]\n" +
+ " [\"zero\",\"one\"] \n" +
+ " [\"six\",\"seven\",\"eight\"]\n" +
+ " [\"six\",\"seven\",\"eight\",\"nine\",\"ten\"]\n" +
+ " [\"six\",\"seven\",\"eight\",\"nine\",\"ten\"]\n" +
+ " [\"eight\",\"nine\",\"ten\"]\n" +
+ " [\"eight\",\"nine\"]")
+// @formatter:on
@UDFType(deterministic = true, stateful = false)
public final class ArraySliceUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayToStrUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayToStrUDF.java b/core/src/main/java/hivemall/tools/array/ArrayToStrUDF.java
new file mode 100644
index 0000000..185abc7
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/array/ArrayToStrUDF.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.array;
+
+import hivemall.utils.hadoop.HiveUtils;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+
+@Description(name = "array_to_str",
+ value = "_FUNC_(array arr [, string sep=',']) - Convert array to string using a sperator",
+ extended = "SELECT array_to_str(array(1,2,3),'-');\n" + "1-2-3")
+@UDFType(deterministic = true, stateful = false)
+public final class ArrayToStrUDF extends GenericUDF {
+
+ private ListObjectInspector listOI;
+ @Nullable
+ private StringObjectInspector sepOI;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 1 && argOIs.length != 2) {
+ throw new UDFArgumentLengthException(
+ "array_to_str(array, string sep) expects one or two arguments: " + argOIs.length);
+ }
+
+ this.listOI = HiveUtils.asListOI(argOIs[0]);
+ if (argOIs.length == 2) {
+ this.sepOI = HiveUtils.asStringOI(argOIs[1]);
+ }
+
+ return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ }
+
+ @Override
+ public String evaluate(DeferredObject[] arguments) throws HiveException {
+ Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+
+ String sep = ",";
+ if (arguments.length == 2) {
+ Object arg1 = arguments[1].get();
+ if (arg1 != null) {
+ sep = sepOI.getPrimitiveJavaObject(arg1);
+ }
+ }
+
+ final StringBuilder buf = new StringBuilder();
+ final int len = listOI.getListLength(arg0);
+ for (int i = 0; i < len; i++) {
+ Object e = listOI.getListElement(arg0, i);
+ if (e != null) {
+ if (i != 0 && buf.length() > 0) {
+ buf.append(sep);
+ }
+ buf.append(e.toString());
+ }
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "array_to_str(" + StringUtils.join(children, ',') + ")";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ArrayUnionUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ArrayUnionUDF.java b/core/src/main/java/hivemall/tools/array/ArrayUnionUDF.java
index 921bbfa..91ae53f 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayUnionUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayUnionUDF.java
@@ -46,7 +46,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.Object
* </pre>
*/
@Description(name = "array_union",
- value = "_FUNC_(array1, array2, ...) - Returns the union of a set of arrays")
+ value = "_FUNC_(array1, array2, ...) - Returns the union of a set of arrays",
+ extended = "SELECT array_union(array(1,2),array(1,2));\n" + "[1,2]\n\n"
+ + "SELECT array_union(array(1,2),array(2,3),array(2,5));\n" + "[1,2,3,5]")
@UDFType(deterministic = true, stateful = false)
public final class ArrayUnionUDF extends GenericUDF {
@@ -95,9 +97,9 @@ public final class ArrayUnionUDF extends GenericUDF {
for (int j = 0, len = oi.getListLength(undeferred); j < len; ++j) {
Object nonStd = oi.getListElement(undeferred, j);
- Object copied = ObjectInspectorUtils.copyToStandardObject(nonStd, elemOI,
+ Object copyed = ObjectInspectorUtils.copyToStandardObject(nonStd, elemOI,
ObjectInspectorCopyOption.WRITABLE);
- objectSet.add(copied);
+ objectSet.add(copyed);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/ConditionalEmitUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/ConditionalEmitUDTF.java b/core/src/main/java/hivemall/tools/array/ConditionalEmitUDTF.java
index a73a06f..89862b9 100644
--- a/core/src/main/java/hivemall/tools/array/ConditionalEmitUDTF.java
+++ b/core/src/main/java/hivemall/tools/array/ConditionalEmitUDTF.java
@@ -55,9 +55,26 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInsp
* table_to_scan_once
* </pre>
*/
+// @formatter:off
@Description(name = "conditional_emit",
value = "_FUNC_(array<boolean> conditions, array<primitive> features)"
- + " - Emit features of a row according to various conditions")
+ + " - Emit features of a row according to various conditions",
+ extended = "WITH input as (\n" +
+ " select array(true, false, true) as conditions, array(\"one\", \"two\", \"three\") as features\n" +
+ " UNION ALL\n" +
+ " select array(true, true, false), array(\"four\", \"five\", \"six\")\n" +
+ ")\n" +
+ "SELECT\n" +
+ " conditional_emit(\n" +
+ " conditions, features\n" +
+ " )\n" +
+ "FROM \n" +
+ " input;\n" +
+ " one\n" +
+ " three\n" +
+ " four\n" +
+ " five")
+// @formatter:on
@UDFType(deterministic = true, stateful = false)
public final class ConditionalEmitUDTF extends GenericUDTF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/FirstElementUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/FirstElementUDF.java b/core/src/main/java/hivemall/tools/array/FirstElementUDF.java
index 957e724..a708529 100644
--- a/core/src/main/java/hivemall/tools/array/FirstElementUDF.java
+++ b/core/src/main/java/hivemall/tools/array/FirstElementUDF.java
@@ -31,7 +31,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* Return the first element in an array.
*/
-@Description(name = "first_element", value = "_FUNC_(x) - Returns the first element in an array ")
+@Description(name = "first_element", value = "_FUNC_(x) - Returns the first element in an array",
+ extended = "SELECT first_element(array('a','b','c'));\n a\n\n"
+ + "SELECT first_element(array());\n NULL")
@UDFType(deterministic = true, stateful = false)
public class FirstElementUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/LastElementUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/LastElementUDF.java b/core/src/main/java/hivemall/tools/array/LastElementUDF.java
index 91050b8..f36a6aa 100644
--- a/core/src/main/java/hivemall/tools/array/LastElementUDF.java
+++ b/core/src/main/java/hivemall/tools/array/LastElementUDF.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* Return the last element in an array.
*/
-@Description(name = "last_element", value = "_FUNC_(x) - Return the last element in an array")
+@Description(name = "last_element", value = "_FUNC_(x) - Return the last element in an array",
+ extended = "SELECT last_element(array('a','b','c'));\n c")
@UDFType(deterministic = true, stateful = false)
public class LastElementUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/SortAndUniqArrayUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/SortAndUniqArrayUDF.java b/core/src/main/java/hivemall/tools/array/SortAndUniqArrayUDF.java
index e9c1cc5..d9f265e 100644
--- a/core/src/main/java/hivemall/tools/array/SortAndUniqArrayUDF.java
+++ b/core/src/main/java/hivemall/tools/array/SortAndUniqArrayUDF.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.io.IntWritable;
@Description(name = "sort_and_uniq_array",
value = "_FUNC_(array<int>) - Takes array<int> and "
+ "returns a sorted array with duplicate elements eliminated",
- extended = "select sort_and_uniq_array(array(3,1,1,-2,10));\n" + "> [-2,1,3,10]")
+ extended = "SELECT sort_and_uniq_array(array(3,1,1,-2,10));\n" + " [-2,1,3,10]")
@UDFType(deterministic = true, stateful = false)
public class SortAndUniqArrayUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/SubarrayEndWithUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/SubarrayEndWithUDF.java b/core/src/main/java/hivemall/tools/array/SubarrayEndWithUDF.java
index 9018117..36f23b5 100644
--- a/core/src/main/java/hivemall/tools/array/SubarrayEndWithUDF.java
+++ b/core/src/main/java/hivemall/tools/array/SubarrayEndWithUDF.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.Text;
@Description(name = "subarray_endwith",
value = "_FUNC_(array<int|text> original, int|text key)"
+ " - Returns an array that ends with the specified key",
- extended = "select subarray_endwith(array(1,2,3,4), 3);\n" + "> [1,2,3]")
+ extended = "SELECT subarray_endwith(array(1,2,3,4), 3);\n" + " [1,2,3]")
@UDFType(deterministic = true, stateful = false)
public class SubarrayEndWithUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/array/SubarrayStartWithUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/array/SubarrayStartWithUDF.java b/core/src/main/java/hivemall/tools/array/SubarrayStartWithUDF.java
index ae0d4fb..95ae6cf 100644
--- a/core/src/main/java/hivemall/tools/array/SubarrayStartWithUDF.java
+++ b/core/src/main/java/hivemall/tools/array/SubarrayStartWithUDF.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.Text;
@Description(name = "subarray_startwith",
value = "_FUNC_(array<int|text> original, int|text key)"
+ " - Returns an array that starts with the specified key",
- extended = "select subarray_startwith(array(1,2,3,4), 2);\n" + "> [2,3,4]")
+ extended = "SELECT subarray_startwith(array(1,2,3,4), 2);\n" + " [2,3,4]")
@UDFType(deterministic = true, stateful = false)
public class SubarrayStartWithUDF extends UDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/bits/BitsORUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/bits/BitsORUDF.java b/core/src/main/java/hivemall/tools/bits/BitsORUDF.java
index d5497b6..40f7b5c 100644
--- a/core/src/main/java/hivemall/tools/bits/BitsORUDF.java
+++ b/core/src/main/java/hivemall/tools/bits/BitsORUDF.java
@@ -38,8 +38,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
@Description(name = "bits_or",
value = "_FUNC_(array<long> b1, array<long> b2, ..) - Returns a logical OR given bitsets",
- extended = "select unbits(bits_or(to_bits(array(1,4)),to_bits(array(2,3))));\n"
- + "> [1,2,3,4]")
+ extended = "SELECT unbits(bits_or(to_bits(array(1,4)),to_bits(array(2,3))));\n"
+ + " [1,2,3,4]")
public final class BitsORUDF extends GenericUDF {
private ListObjectInspector[] _listOIs;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/bits/ToBitsUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/bits/ToBitsUDF.java b/core/src/main/java/hivemall/tools/bits/ToBitsUDF.java
index 599d90a..c33bd31 100644
--- a/core/src/main/java/hivemall/tools/bits/ToBitsUDF.java
+++ b/core/src/main/java/hivemall/tools/bits/ToBitsUDF.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.io.LongWritable;
@Description(name = "to_bits",
value = "_FUNC_(int[] indexes) - Returns an bitset representation if the given indexes in long[]",
- extended = "select to_bits(array(1,2,3,128));\n" + "> [14,-9223372036854775808]")
+ extended = "SELECT to_bits(array(1,2,3,128));\n" + " [14,-9223372036854775808]")
@UDFType(deterministic = true, stateful = false)
public final class ToBitsUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/bits/UnBitsUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/bits/UnBitsUDF.java b/core/src/main/java/hivemall/tools/bits/UnBitsUDF.java
index e3f6bae..0ec3762 100644
--- a/core/src/main/java/hivemall/tools/bits/UnBitsUDF.java
+++ b/core/src/main/java/hivemall/tools/bits/UnBitsUDF.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.io.LongWritable;
@Description(name = "unbits",
value = "_FUNC_(long[] bitset) - Returns an long array of the give bitset representation",
- extended = "select unbits(to_bits(array(1,4,2,3)));\n" + "> [1,2,3,4]")
+ extended = "SELECT unbits(to_bits(array(1,4,2,3)));\n" + " [1,2,3,4]")
@UDFType(deterministic = true, stateful = false)
public final class UnBitsUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/compress/DeflateUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/compress/DeflateUDF.java b/core/src/main/java/hivemall/tools/compress/DeflateUDF.java
index 4c82387..410ffd6 100644
--- a/core/src/main/java/hivemall/tools/compress/DeflateUDF.java
+++ b/core/src/main/java/hivemall/tools/compress/DeflateUDF.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.io.Text;
@Description(name = "deflate", value = "_FUNC_(TEXT data [, const int compressionLevel]) - "
+ "Returns a compressed BINARY object by using Deflater. The compression level must be in range [-1,9]",
- extended = "select base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));\n" + "> AA+=kaIM|WTt!+wbGAA")
+ extended = "SELECT base91(deflate('aaaaaaaaaaaaaaaabbbbccc'));\n" + " AA+=kaIM|WTt!+wbGAA")
@UDFType(deterministic = true, stateful = false)
public final class DeflateUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/compress/InflateUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/compress/InflateUDF.java b/core/src/main/java/hivemall/tools/compress/InflateUDF.java
index e2c4cc2..883d432 100644
--- a/core/src/main/java/hivemall/tools/compress/InflateUDF.java
+++ b/core/src/main/java/hivemall/tools/compress/InflateUDF.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.io.Text;
@Description(name = "inflate",
value = "_FUNC_(BINARY compressedData) - Returns a decompressed STRING by using Inflater",
- extended = "select inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));\n"
- + "> aaaaaaaaaaaaaaaabbbbccc")
+ extended = "SELECT inflate(unbase91(base91(deflate('aaaaaaaaaaaaaaaabbbbccc'))));\n"
+ + " aaaaaaaaaaaaaaaabbbbccc")
@UDFType(deterministic = true, stateful = false)
public final class InflateUDF extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/datetime/SessionizeUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/datetime/SessionizeUDF.java b/core/src/main/java/hivemall/tools/datetime/SessionizeUDF.java
new file mode 100644
index 0000000..8235842
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/datetime/SessionizeUDF.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.datetime;
+
+import java.util.UUID;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+//@formatter:off
+@Description(name = "sessionize",
+ value = "_FUNC_(long timeInSec, long thresholdInSec [, String subject])"
+ + "- Returns a UUID string of a session.",
+ extended = "SELECT \n" +
+ " sessionize(time, 3600, ip_addr) as session_id, \n" +
+ " time, ip_addr\n" +
+ "FROM (\n" +
+ " SELECT time, ipaddr \n" +
+ " FROM weblog \n" +
+ " DISTRIBUTE BY ip_addr, time SORT BY ip_addr, time DESC\n" +
+ ") t1")
+//@formatter:on
+@UDFType(deterministic = false, stateful = true)
+public final class SessionizeUDF extends UDF {
+
+ private long lastTime;
+ @Nullable
+ private Text lastSubject;
+ @Nonnull
+ private final Text sessionId = new Text();
+
+ @Nullable
+ public Text evaluate(@Nullable LongWritable time, @Nullable LongWritable threshold) {
+ if (time == null || threshold == null) {
+ return null;
+ }
+
+ final long thisTime = time.get();
+ final long diff = thisTime - lastTime;
+ if (diff < threshold.get()) {
+ this.lastTime = thisTime;
+ return sessionId;
+ }
+
+ sessionId.set(UUID.randomUUID().toString());
+ this.lastTime = time.get();
+ return sessionId;
+ }
+
+ @Nullable
+ public Text evaluate(@Nullable LongWritable time, @Nullable LongWritable threshold,
+ @Nullable Text subject) {
+ if (time == null || threshold == null || subject == null) {
+ return null;
+ }
+
+ if (subject.equals(lastSubject)) {
+ final long thisTime = time.get();
+ final long diff = thisTime - lastTime;
+ if (diff < threshold.get()) {
+ this.lastTime = thisTime;
+ return sessionId;
+ }
+ } else {
+ if (lastSubject == null) {
+ lastSubject = new Text(subject);
+ } else {
+ lastSubject.set(subject);
+ }
+ }
+
+ sessionId.set(UUID.randomUUID().toString());
+ this.lastTime = time.get();
+ return sessionId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/49496032/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/json/FromJsonUDF.java b/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
index 36c29cc..97409dd 100644
--- a/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
+++ b/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
@@ -43,9 +43,47 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.Text;
import org.apache.hive.hcatalog.data.HCatRecordObjectInspectorFactory;
+// @formatter:off
@Description(name = "from_json",
value = "_FUNC_(string jsonString, const string returnTypes [, const array<string>|const string columnNames])"
- + " - Return Hive object.")
+ + " - Return Hive object.",
+ extended = "SELECT\n" +
+ " from_json(\n" +
+ " '{ \"person\" : { \"name\" : \"makoto\" , \"age\" : 37 } }',\n" +
+ " 'struct<name:string,age:int>', \n" +
+ " array('person')\n" +
+ " ),\n" +
+ " from_json(\n" +
+ " '[0.1,1.1,2.2]',\n" +
+ " 'array<double>'\n" +
+ " ),\n" +
+ " from_json(to_json(\n" +
+ " ARRAY(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"osaka\")\n" +
+ " )\n" +
+ " ),'array<struct<country:string,city:string>>'),\n" +
+ " from_json(to_json(\n" +
+ " ARRAY(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"osaka\")\n" +
+ " ),\n" +
+ " array('city')\n" +
+ " ), 'array<struct<country:string,city:string>>'),\n" +
+ " from_json(to_json(\n" +
+ " ARRAY(\n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"tokyo\"), \n" +
+ " NAMED_STRUCT(\"country\", \"japan\", \"city\", \"osaka\")\n" +
+ " )\n" +
+ " ),'array<struct<city:string>>');\n"
+ + "```\n\n" +
+ "```\n" +
+ " {\"name\":\"makoto\",\"age\":37}\n" +
+ " [0.1,1.1,2.2]\n" +
+ " [{\"country\":\"japan\",\"city\":\"tokyo\"},{\"country\":\"japan\",\"city\":\"osaka\"}]\n" +
+ " [{\"country\":\"japan\",\"city\":\"tokyo\"},{\"country\":\"japan\",\"city\":\"osaka\"}]\n" +
+ " [{\"city\":\"tokyo\"},{\"city\":\"osaka\"}]")
+//@formatter:on
@UDFType(deterministic = true, stateful = false)
public final class FromJsonUDF extends GenericUDF {