You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2017/02/10 11:49:43 UTC
incubator-hivemall git commit: Close #43: Added rownum() UDF
Repository: incubator-hivemall
Updated Branches:
refs/heads/master 85f8e173a -> 389a9e331
Close #43: Added rownum() UDF
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/389a9e33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/389a9e33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/389a9e33
Branch: refs/heads/master
Commit: 389a9e331a1fdb2884d7b184b8ed63e510d9acc0
Parents: 85f8e17
Author: myui <yu...@gmail.com>
Authored: Fri Feb 10 20:49:06 2017 +0900
Committer: myui <yu...@gmail.com>
Committed: Fri Feb 10 20:49:06 2017 +0900
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE | 6 +-
.../hivemall/tools/mapred/RowNumberUDF.java | 69 ++++++++++++++++++++
resources/ddl/define-all-as-permanent.hive | 3 +
resources/ddl/define-all.hive | 3 +
4 files changed, 80 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/389a9e33/.github/PULL_REQUEST_TEMPLATE
----------------------------------------------------------------------
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
index 04f3004..361d613 100644
--- a/.github/PULL_REQUEST_TEMPLATE
+++ b/.github/PULL_REQUEST_TEMPLATE
@@ -6,10 +6,14 @@
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
-### What is the Jira issue?
+## What is the Jira issue?
(Put link here and add [HIVEMALL-*Jira number*] in PR title, e.g., [HIVEMALL-533])
## How was this patch tested?
(Please explain how this patch was tested. e.g., unit tests, integration tests, manual tests)
+
+## How to use this feature?
+
+(Please remove this section if not needed)
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/389a9e33/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java b/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
new file mode 100644
index 0000000..59f64ba
--- /dev/null
+++ b/core/src/main/java/hivemall/tools/mapred/RowNumberUDF.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.mapred;
+
+import hivemall.utils.hadoop.HadoopUtils;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.io.LongWritable;
+
+@Description(name = "rownum", value = "_FUNC_() - Returns a generated row number in long",
+ extended = "returns sprintf(`%d%04d`,sequence,taskId) as long")
+@UDFType(deterministic = false, stateful = true)
+public final class RowNumberUDF extends UDF {
+
+ private long sequence;
+ private int taskId;
+ @Nonnull
+ private final LongWritable result;
+
+ public RowNumberUDF() {
+ this.sequence = 0L;
+ this.taskId = -1;
+ this.result = new LongWritable(Double.doubleToLongBits(Double.NaN));
+ }
+
+ @Nonnull
+ public LongWritable evaluate() throws HiveException {
+ if (taskId == -1) {
+ this.taskId = HadoopUtils.getTaskId() + 1;
+ if (taskId > 9999) {
+ throw new HiveException("TaskId out of range `" + taskId
+ + "`. rownum() supports 9999 tasks at max");
+ }
+ }
+ sequence++;
+
+ String rowid = String.format("%d%04d", sequence, taskId);
+ final long l;
+ try {
+ l = Long.parseLong(rowid);
+ } catch (NumberFormatException e) {
+ throw new HiveException("failed to parse `" + rowid + "` as long", e);
+ }
+
+ result.set(l);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/389a9e33/resources/ddl/define-all-as-permanent.hive
----------------------------------------------------------------------
diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive
index 33a15a1..f9f9fd8 100644
--- a/resources/ddl/define-all-as-permanent.hive
+++ b/resources/ddl/define-all-as-permanent.hive
@@ -477,6 +477,9 @@ CREATE FUNCTION jobid as 'hivemall.tools.mapred.JobIdUDF' USING JAR '${hivemall_
DROP FUNCTION IF EXISTS rowid;
CREATE FUNCTION rowid as 'hivemall.tools.mapred.RowIdUDF' USING JAR '${hivemall_jar}';
+DROP FUNCTION IF EXISTS rownum;
+CREATE FUNCTION rownum as 'hivemall.tools.mapred.RowNumberUDF' USING JAR '${hivemall_jar}';
+
DROP FUNCTION IF EXISTS distcache_gets;
CREATE FUNCTION distcache_gets as 'hivemall.tools.mapred.DistributedCacheLookupUDF' USING JAR '${hivemall_jar}';
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/389a9e33/resources/ddl/define-all.hive
----------------------------------------------------------------------
diff --git a/resources/ddl/define-all.hive b/resources/ddl/define-all.hive
index 9cc348f..6e0f911 100644
--- a/resources/ddl/define-all.hive
+++ b/resources/ddl/define-all.hive
@@ -473,6 +473,9 @@ create temporary function jobid as 'hivemall.tools.mapred.JobIdUDF';
drop temporary function if exists rowid;
create temporary function rowid as 'hivemall.tools.mapred.RowIdUDF';
+drop temporary function if exists rownum;
+create temporary function rownum as 'hivemall.tools.mapred.RowNumberUDF';
+
drop temporary function if exists distcache_gets;
create temporary function distcache_gets as 'hivemall.tools.mapred.DistributedCacheLookupUDF';