You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/04 10:10:55 UTC
spark git commit: [SPARK-8270][SQL] levenshtein distance
Repository: spark
Updated Branches:
refs/heads/master f35b0c343 -> 6b3574e68
[SPARK-8270][SQL] levenshtein distance
Jira: https://issues.apache.org/jira/browse/SPARK-8270
Info: I can not build the latest master, it stucks during the build process: `[INFO] Dependency-reduced POM written at: /Users/tarek/test/spark/bagel/dependency-reduced-pom.xml`
Author: Tarek Auel <ta...@googlemail.com>
Closes #7214 from tarekauel/SPARK-8270 and squashes the following commits:
ab348b9 [Tarek Auel] Merge branch 'master' into SPARK-8270
a2ad318 [Tarek Auel] [SPARK-8270] changed order of fields
d91b12c [Tarek Auel] [SPARK-8270] python fix
adbd075 [Tarek Auel] [SPARK-8270] fixed typo
23185c9 [Tarek Auel] [SPARK-8270] levenshtein distance
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6b3574e6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6b3574e6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6b3574e6
Branch: refs/heads/master
Commit: 6b3574e68704d58ba41efe0ea4fe928cc166afcd
Parents: f35b0c3
Author: Tarek Auel <ta...@googlemail.com>
Authored: Sat Jul 4 01:10:52 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Sat Jul 4 01:10:52 2015 -0700
----------------------------------------------------------------------
python/pyspark/sql/functions.py | 14 +++++++++
.../catalyst/analysis/FunctionRegistry.scala | 1 +
.../catalyst/expressions/stringOperations.scala | 32 ++++++++++++++++++++
.../expressions/StringFunctionsSuite.scala | 9 ++++++
.../scala/org/apache/spark/sql/functions.scala | 23 +++++++++++---
.../spark/sql/DataFrameFunctionsSuite.scala | 6 ++++
6 files changed, 81 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 69e563e..49dd033 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -325,6 +325,20 @@ def explode(col):
@ignore_unicode_prefix
@since(1.5)
+def levenshtein(left, right):
+ """Computes the Levenshtein distance of the two given strings.
+
+ >>> df0 = sqlContext.createDataFrame([('kitten', 'sitting',)], ['l', 'r'])
+ >>> df0.select(levenshtein('l', 'r').alias('d')).collect()
+ [Row(d=3)]
+ """
+ sc = SparkContext._active_spark_context
+ jc = sc._jvm.functions.levenshtein(_to_java_column(left), _to_java_column(right))
+ return Column(jc)
+
+
+@ignore_unicode_prefix
+@since(1.5)
def md5(col):
"""Calculates the MD5 digest and returns the value as a 32 character hex string.
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index e249b58..92a50e7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -163,6 +163,7 @@ object FunctionRegistry {
expression[Lower]("lcase"),
expression[Lower]("lower"),
expression[StringLength]("length"),
+ expression[Levenshtein]("levenshtein"),
expression[Substring]("substr"),
expression[Substring]("substring"),
expression[UnBase64]("unbase64"),
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 154ac35..6de4062 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
import java.util.regex.Pattern
+import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.catalyst.analysis.UnresolvedException
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.types._
@@ -300,6 +301,37 @@ case class StringLength(child: Expression) extends UnaryExpression with ExpectsI
}
/**
+ * A function that return the Levenshtein distance between the two given strings.
+ */
+case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression
+ with ExpectsInputTypes {
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType)
+
+ override def dataType: DataType = IntegerType
+
+ override def eval(input: InternalRow): Any = {
+ val leftValue = left.eval(input)
+ if (leftValue == null) {
+ null
+ } else {
+ val rightValue = right.eval(input)
+ if(rightValue == null) {
+ null
+ } else {
+ StringUtils.getLevenshteinDistance(leftValue.toString, rightValue.toString)
+ }
+ }
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
+ val stringUtils = classOf[StringUtils].getName
+ nullSafeCodeGen(ctx, ev, (res, left, right) =>
+ s"$res = $stringUtils.getLevenshteinDistance($left.toString(), $right.toString());")
+ }
+}
+
+/**
* Returns the numeric value of the first character of str.
*/
case class Ascii(child: Expression) extends UnaryExpression with ExpectsInputTypes {
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala
index 468df20..1efbe1a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala
@@ -274,4 +274,13 @@ class StringFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Decode(Literal.create(null, BinaryType), Literal("utf-8")), null)
checkEvaluation(Decode(b, Literal.create(null, StringType)), null, create_row(null))
}
+
+ test("Levenshtein distance") {
+ checkEvaluation(Levenshtein(Literal.create(null, StringType), Literal("")), null)
+ checkEvaluation(Levenshtein(Literal(""), Literal.create(null, StringType)), null)
+ checkEvaluation(Levenshtein(Literal(""), Literal("")), 0)
+ checkEvaluation(Levenshtein(Literal("abc"), Literal("abc")), 0)
+ checkEvaluation(Levenshtein(Literal("kitten"), Literal("sitting")), 3)
+ checkEvaluation(Levenshtein(Literal("frog"), Literal("fog")), 1)
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index b63c6ee..e4109da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -1580,22 +1580,37 @@ object functions {
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Computes the length of a given string value
- *
+ * Computes the length of a given string value.
+ *
* @group string_funcs
* @since 1.5.0
*/
def strlen(e: Column): Column = StringLength(e.expr)
/**
- * Computes the length of a given string column
- *
+ * Computes the length of a given string column.
+ *
* @group string_funcs
* @since 1.5.0
*/
def strlen(columnName: String): Column = strlen(Column(columnName))
/**
+ * Computes the Levenshtein distance of the two given strings.
+ * @group string_funcs
+ * @since 1.5.0
+ */
+ def levenshtein(l: Column, r: Column): Column = Levenshtein(l.expr, r.expr)
+
+ /**
+ * Computes the Levenshtein distance of the two given strings.
+ * @group string_funcs
+ * @since 1.5.0
+ */
+ def levenshtein(leftColumnName: String, rightColumnName: String): Column =
+ levenshtein(Column(leftColumnName), Column(rightColumnName))
+
+ /**
* Computes the numeric value of the first character of the specified string value.
*
* @group string_funcs
http://git-wip-us.apache.org/repos/asf/spark/blob/6b3574e6/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index bd9fa40..bc455a9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -226,6 +226,12 @@ class DataFrameFunctionsSuite extends QueryTest {
})
}
+ test("Levenshtein distance") {
+ val df = Seq(("kitten", "sitting"), ("frog", "fog")).toDF("l", "r")
+ checkAnswer(df.select(levenshtein("l", "r")), Seq(Row(3), Row(1)))
+ checkAnswer(df.selectExpr("levenshtein(l, r)"), Seq(Row(3), Row(1)))
+ }
+
test("string ascii function") {
val df = Seq(("abc", "")).toDF("a", "b")
checkAnswer(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org