You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2016/05/17 09:51:33 UTC
[02/14] jena git commit: Implemented fn:normalized-unicode
Implemented fn:normalized-unicode
First implementation of fn:normalized-unicode with tests. Some doubt remain for part of the implementation.
Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/62394644
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/62394644
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/62394644
Branch: refs/heads/master
Commit: 62394644c624f637f5ff7084b151cc404802e569
Parents: 3b00217
Author: ales004 <ci...@hotmail.com>
Authored: Mon May 2 22:00:22 2016 +0200
Committer: ales004 <ci...@hotmail.com>
Committed: Mon May 2 22:00:22 2016 +0200
----------------------------------------------------------------------
.../jena/sparql/expr/nodevalue/XSDFuncOp.java | 37 +++++++++++++
.../jena/sparql/function/StandardFunctions.java | 2 +
.../library/FN_StrNormalizeUnicode.java | 57 ++++++++++++++++++++
.../apache/jena/sparql/expr/TestFunctions.java | 26 +++++++++
4 files changed, 122 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jena/blob/62394644/jena-arq/src/main/java/org/apache/jena/sparql/expr/nodevalue/XSDFuncOp.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/expr/nodevalue/XSDFuncOp.java b/jena-arq/src/main/java/org/apache/jena/sparql/expr/nodevalue/XSDFuncOp.java
index 7b03d79..501f4ec 100644
--- a/jena-arq/src/main/java/org/apache/jena/sparql/expr/nodevalue/XSDFuncOp.java
+++ b/jena-arq/src/main/java/org/apache/jena/sparql/expr/nodevalue/XSDFuncOp.java
@@ -33,6 +33,7 @@ import static org.apache.jena.sparql.expr.nodevalue.NumericType.OP_INTEGER ;
import java.math.BigDecimal ;
import java.math.BigInteger ;
+import java.text.Normalizer;
import java.util.ArrayList;
import java.util.HashSet ;
import java.util.List ;
@@ -715,10 +716,46 @@ public class XSDFuncOp
String str = v.asString() ;
if(str == "" )
return NodeValue.nvEmptyString;
+ // is it possible that str is null?
str = str.trim().replaceAll("\\s+"," ");
return NodeValue.makeString(str) ;
}
+ public static NodeValue strNormalizeUnicode(NodeValue v1, NodeValue v2) {
+ String normalizationFormStr = "nfc";
+ if(v2 != null)
+ normalizationFormStr = v2.asNode().getLiteralLexicalForm().toLowerCase();
+
+ String inputString = v1.asString();
+ if(normalizationFormStr.isEmpty())
+ return NodeValue.makeString(inputString);
+ // is it possible that normalizationFormStr is null?
+
+ Normalizer.Form normalizationForm = Normalizer.Form.NFC;
+ switch(normalizationFormStr)
+ {
+ case "nfd":
+ normalizationForm = Normalizer.Form.NFD;
+ break;
+ case "nfkd":
+ normalizationForm = Normalizer.Form.NFKD;
+ break;
+ case "nfkc":
+ normalizationForm = Normalizer.Form.NFKC;
+ break;
+ case "nfc":
+ normalizationForm = Normalizer.Form.NFC;
+ break;
+ case "fully-normalized":
+ // not fully understood how to implement the fully-normalized normalization form.
+ throw new ExprEvalTypeException("The fully-normalized normalization form is not supported.");
+ default:
+ throw new ExprEvalTypeException("Unrecognized normalization form "+normalizationFormStr+". Supported normalization forms: NFC,NFD,NFKD and NFKC.");
+ }
+
+ return NodeValue.makeString(Normalizer.normalize(inputString,normalizationForm));
+ }
+
public static NumericType classifyNumeric(String fName, NodeValue nv1, NodeValue nv2) {
if ( !nv1.isNumber() )
throw new ExprEvalTypeException("Not a number (first arg to " + fName + "): " + nv1) ;
http://git-wip-us.apache.org/repos/asf/jena/blob/62394644/jena-arq/src/main/java/org/apache/jena/sparql/function/StandardFunctions.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/function/StandardFunctions.java b/jena-arq/src/main/java/org/apache/jena/sparql/function/StandardFunctions.java
index c45efff..9f9eb82 100644
--- a/jena-arq/src/main/java/org/apache/jena/sparql/function/StandardFunctions.java
+++ b/jena-arq/src/main/java/org/apache/jena/sparql/function/StandardFunctions.java
@@ -102,6 +102,8 @@ public class StandardFunctions
// fn:normalize-space
add(registry,xfn+"normalize-space", FN_StrNormalizeSpace.class);
// fn:normalize-unicode
+ add(registry,xfn+"normalize-unicode", FN_StrNormalizeUnicode.class);
+
add(registry, xfn+"upper-case", FN_StrUpperCase.class) ;
add(registry, xfn+"lower-case", FN_StrLowerCase.class) ;
add(registry, xfn+"contains", FN_StrContains.class) ;
http://git-wip-us.apache.org/repos/asf/jena/blob/62394644/jena-arq/src/main/java/org/apache/jena/sparql/function/library/FN_StrNormalizeUnicode.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/function/library/FN_StrNormalizeUnicode.java b/jena-arq/src/main/java/org/apache/jena/sparql/function/library/FN_StrNormalizeUnicode.java
new file mode 100644
index 0000000..e7a5813
--- /dev/null
+++ b/jena-arq/src/main/java/org/apache/jena/sparql/function/library/FN_StrNormalizeUnicode.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jena.sparql.function.library;
+
+import org.apache.jena.atlas.lib.Lib;
+import org.apache.jena.query.QueryBuildException;
+import org.apache.jena.sparql.expr.ExprEvalException;
+import org.apache.jena.sparql.expr.ExprList;
+import org.apache.jena.sparql.expr.NodeValue;
+import org.apache.jena.sparql.expr.nodevalue.XSDFuncOp;
+import org.apache.jena.sparql.function.FunctionBase;
+
+import java.util.List;
+
+public class FN_StrNormalizeUnicode extends FunctionBase
+{
+ public FN_StrNormalizeUnicode() { super() ; }
+
+ @Override
+ public void checkBuild(String uri, ExprList args)
+ {
+ if ( args.size() != 1 && args.size() != 2 )
+ throw new QueryBuildException("Function '"+ Lib.className(this)+"' takes one or two arguments") ;
+ }
+ @Override
+ public NodeValue exec(List<NodeValue> args)
+ {
+ if ( args.size() > 2 )
+ throw new ExprEvalException("substring: Wrong number of arguments: "+
+ args.size()+" : [wanted 1 or 2]") ;
+
+ NodeValue v1 = args.get(0) ;
+
+ if ( args.size() == 2 )
+ {
+ NodeValue v2 = args.get(1) ;
+ return XSDFuncOp.strNormalizeUnicode(v1, v2) ;
+ }
+
+ return XSDFuncOp.strNormalizeUnicode(v1, null) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/62394644/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestFunctions.java
----------------------------------------------------------------------
diff --git a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestFunctions.java b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestFunctions.java
index 27679f3..cf38e46 100644
--- a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestFunctions.java
+++ b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestFunctions.java
@@ -187,6 +187,32 @@ public class TestFunctions
@Test public void exprStrNormalizeSpace2() { test("fn:normalize-space(' Aaa ')",NodeValue.makeString("Aaa")) ; }
@Test public void exprStrNormalizeSpace3() { test("fn:normalize-space('A a a a a ')",NodeValue.makeString("A a a a a")) ; }
+ // from http://www.unicode.org/reports/tr15/
+ //l
+ @Test public void exprStrNormalizeUnicode0() { test("fn:normalize-unicode('�ffin','nfd')",NodeValue.makeString("A\u0308ffin")) ; }
+ @Test public void exprStrNormalizeUnicode1() { test("fn:normalize-unicode('�ffin','nfc')",NodeValue.makeString("�ffin")) ; }
+ //m
+ @Test public void exprStrNormalizeUnicode2() { test("fn:normalize-unicode('�\\uFB03n','nfd')",NodeValue.makeString("A\u0308\ufb03n")) ; }
+ @Test public void exprStrNormalizeUnicode3() { test("fn:normalize-unicode('�\\uFB03n','nfc')",NodeValue.makeString("�\ufb03n")) ; }
+ //n
+ @Test public void exprStrNormalizeUnicode4() { test("fn:normalize-unicode('Henry IV','nfd')",NodeValue.makeString("Henry IV")) ; }
+ @Test public void exprStrNormalizeUnicode5() { test("fn:normalize-unicode('Henry IV','nfc')",NodeValue.makeString("Henry IV")) ; }
+ //l'
+ @Test public void exprStrNormalizeUnicode6() { test("fn:normalize-unicode('�ffin','nfkd')",NodeValue.makeString("A\u0308ffin")) ; }
+ @Test public void exprStrNormalizeUnicode7() { test("fn:normalize-unicode('�ffin','nfkc')",NodeValue.makeString("�ffin")) ; }
+ // r
+ @Test public void exprStrNormalizeUnicode8() { test("fn:normalize-unicode('hw_ka + hw_ten','nfd')",NodeValue.makeString("hw_ka + hw_ten")) ; }
+ @Test public void exprStrNormalizeUnicode9() { test("fn:normalize-unicode('hw_ka + hw_ten','nfc')",NodeValue.makeString("hw_ka + hw_ten")) ; }
+ // r'
+ // not sure why test 10 and 11 are not passing... maybe because it is not fully clear what they are testing.
+ @Test public void exprStrNormalizeUnicode10() { test("fn:normalize-unicode('hw_ka + hw_ten','nfkd')",NodeValue.makeString("ka + ten")) ; }
+ @Test public void exprStrNormalizeUnicode11() { test("fn:normalize-unicode('hw_ka + hw_ten','nfkc')",NodeValue.makeString("ga")) ; }
+ // empty argument <-> returns the input string
+ @Test public void exprStrNormalizeUnicode12() { test("fn:normalize-unicode('some word','')",NodeValue.makeString("some word")) ; }
+ // one argument <-> NFC
+ @Test public void exprStrNormalizeUnicode13() { test("fn:normalize-unicode('�ffin')",NodeValue.makeString("�ffin")) ; }
+
+
@Test public void exprReplace01() { test("REPLACE('abc', 'b', 'Z')", NodeValue.makeString("aZc")) ; }
@Test public void exprReplace02() { test("REPLACE('abc', 'b.', 'Z')", NodeValue.makeString("aZ")) ; }
@Test public void exprReplace03() { test("REPLACE('abcbd', 'b.', 'Z')", NodeValue.makeString("aZZ")) ; }