You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2018/01/17 18:51:38 UTC
[3/6] jena git commit: added opts for fragment size and joining
contiguous fragments
added opts for fragment size and joining contiguous fragments
Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/d0e5bd2f
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/d0e5bd2f
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/d0e5bd2f
Branch: refs/heads/master
Commit: d0e5bd2fb78e5a11832c57f9f1680977fd0b68a6
Parents: 99c2aa2
Author: Chris Tomlinson <ct...@moonvine.org>
Authored: Wed Jan 10 09:01:35 2018 -0600
Committer: Chris Tomlinson <ct...@moonvine.org>
Committed: Wed Jan 10 09:01:35 2018 -0600
----------------------------------------------------------------------
.../apache/jena/query/text/TextIndexLucene.java | 25 +++++++++++++++-----
1 file changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jena/blob/d0e5bd2f/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
----------------------------------------------------------------------
diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
index 299bf69..3eacfbe 100644
--- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
+++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
@@ -61,6 +61,7 @@ import org.apache.lucene.search.ScoreDoc ;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
+import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TextFragment ;
import org.apache.lucene.store.Directory ;
@@ -460,10 +461,12 @@ public class TextIndexLucene implements TextIndex {
class HighlightOpts {
int maxFrags = 3;
+ int fragSize = 128;
String start = RIGHT_ARROW;
String end = LEFT_ARROW;
String fragSep = DIVIDES;
- boolean join = true;
+ boolean joinHi = true;
+ boolean joinFrags = true;
public HighlightOpts(String optStr) {
String[] opts = optStr.trim().split("\\|");
@@ -473,16 +476,25 @@ public class TextIndexLucene implements TextIndex {
try {
maxFrags = Integer.parseInt(opt.substring(2));
} catch (Exception ex) { }
+ } else if (opt.startsWith("z:")) {
+ try {
+ fragSize = Integer.parseInt(opt.substring(2));
+ } catch (Exception ex) { }
} else if (opt.startsWith("s:")) {
start = opt.substring(2);
} else if (opt.startsWith("e:")) {
end = opt.substring(2);
} else if (opt.startsWith("f:")) {
fragSep = opt.substring(2);
- } else if (opt.startsWith("j:")) {
- String v = opt.substring(2);
+ } else if (opt.startsWith("jh:")) {
+ String v = opt.substring(3);
+ if ("n".equals(v)) {
+ joinHi = false;
+ }
+ } else if (opt.startsWith("jf:")) {
+ String v = opt.substring(3);
if ("n".equals(v)) {
- join = false;
+ joinFrags = false;
}
}
}
@@ -494,7 +506,7 @@ public class TextIndexLucene implements TextIndex {
String rez = "";
for (TextFragment f : frags) {
- String s = opts.join ? f.toString().replaceAll(opts.end+Z_MORE_SEPS+opts.start, "$1") : f.toString();
+ String s = opts.joinHi ? f.toString().replaceAll(opts.end+Z_MORE_SEPS+opts.start, "$1") : f.toString();
rez += sep + s;
sep = opts.fragSep;
}
@@ -510,6 +522,7 @@ public class TextIndexLucene implements TextIndex {
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(opts.start, opts.end);
Highlighter highlighter = new Highlighter(formatter, new QueryScorer(query));
+ highlighter.setTextFragmenter(new SimpleFragmenter(opts.fragSize));
for ( ScoreDoc sd : sDocs ) {
Document doc = indexSearcher.doc(sd.doc) ;
@@ -521,7 +534,7 @@ public class TextIndexLucene implements TextIndex {
if (lexical != null) {
String docLang = doc.get(docDef.getLangField()) ;
TokenStream tokenStream = analyzer.tokenStream(field, lexical);
- TextFragment[] frags = highlighter.getBestTextFragments(tokenStream, lexical, false, opts.maxFrags);
+ TextFragment[] frags = highlighter.getBestTextFragments(tokenStream, lexical, opts.joinFrags, opts.maxFrags);
String rez = frags2string(frags, opts);
literal = NodeFactory.createLiteral(rez, docLang);