You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by co...@apache.org on 2018/12/05 20:14:22 UTC
[1/2] jena git commit: Merge dev NoIndex
Repository: jena
Updated Branches:
refs/heads/master 0efda76f1 -> 31995c78a
Merge dev NoIndex
Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/b757303a
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/b757303a
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/b757303a
Branch: refs/heads/master
Commit: b757303a205386730888e00d0622ee7b80bb1d3b
Parents: 0efda76
Author: Code Ferret <co...@apache.org>
Authored: Wed Dec 5 10:42:17 2018 -0600
Committer: Code Ferret <co...@apache.org>
Committed: Wed Dec 5 10:42:17 2018 -0600
----------------------------------------------------------------------
.../jena/query/text/EntityDefinition.java | 10 +++++
.../apache/jena/query/text/TextIndexLucene.java | 41 ++++++++++----------
.../assembler/EntityDefinitionAssembler.java | 14 +++++++
.../jena/query/text/assembler/TextVocab.java | 1 +
.../text/TestTextMultilingualEnhancements.java | 2 +-
jena-text/src/test/resources/log4j.properties | 3 ++
6 files changed, 50 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java
----------------------------------------------------------------------
diff --git a/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java b/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java
index 39457d8..d3a2d86 100644
--- a/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java
+++ b/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java
@@ -36,6 +36,7 @@ public class EntityDefinition {
private final Map<Node, String> predicateToField = new HashMap<>() ;
private final Map<String, Analyzer> fieldToAnalyzer = new HashMap<>() ;
private final ListMultimap<String, Node> fieldToPredicate = ArrayListMultimap.create() ;
+ private final Map<String, Boolean> fieldToNoIndex = new HashMap<>() ;
private final Collection<String> fields = Collections.unmodifiableCollection(fieldToPredicate.keys()) ;
// private final Collection<String> fields =
// Collections.unmodifiableCollection(fieldToPredicate.keySet()) ;
@@ -148,6 +149,15 @@ public class EntityDefinition {
public Analyzer getAnalyzer(String field) {
return fieldToAnalyzer.get(field);
}
+
+ public void setNoIndex(String field, boolean b) {
+ fieldToNoIndex.put(field, b);
+ }
+
+ public boolean getNoIndex(String field) {
+ Boolean b = fieldToNoIndex.get(field);
+ return b != null ? b : false;
+ }
public String getPrimaryField() {
return primaryField ;
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
----------------------------------------------------------------------
diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
index 1ddce70..b217c8a 100644
--- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
+++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java
@@ -101,6 +101,7 @@ public class TextIndexLucene implements TextIndex {
private final String queryParserType ;
private final FieldType ftText ;
private final FieldType ftTextNotStored ; // used for lang derived fields
+ private final FieldType ftTextStoredNoIndex ; // used for lang derived fields
private final boolean isMultilingual ;
private final boolean ignoreIndexErrors ;
@@ -161,6 +162,10 @@ public class TextIndexLucene implements TextIndex {
this.ftText = config.isValueStored() ? TextField.TYPE_STORED : TextField.TYPE_NOT_STORED ;
// the following is used for lang derived fields
this.ftTextNotStored = TextField.TYPE_NOT_STORED ;
+ this.ftTextStoredNoIndex = new FieldType();
+ this.ftTextStoredNoIndex.setIndexOptions(IndexOptions.NONE);
+ this.ftTextStoredNoIndex.setStored(true);
+ this.ftTextStoredNoIndex.freeze();
if (config.isValueStored() && docDef.getLangField() == null)
log.warn("Values stored but langField not set. Returned values will not have language tag or datatype.");
@@ -343,7 +348,10 @@ public class TextIndexLucene implements TextIndex {
String uidField = docDef.getUidField() ;
for ( Entry<String, Object> e : entity.getMap().entrySet() ) {
- doc.add( new Field(e.getKey(), (String) e.getValue(), ftText) );
+ String field = e.getKey();
+ String value = (String) e.getValue();
+ FieldType ft = (docDef.getNoIndex(field)) ? ftTextStoredNoIndex : ftText ;
+ doc.add( new Field(field, value, ft) );
if (langField != null) {
String lang = entity.getLanguage();
RDFDatatype datatype = entity.getDatatype();
@@ -351,12 +359,12 @@ public class TextIndexLucene implements TextIndex {
doc.add(new Field(langField, lang, StringField.TYPE_STORED));
if (this.isMultilingual) {
// add a field that uses a language-specific analyzer via MultilingualAnalyzer
- doc.add(new Field(e.getKey() + "_" + lang, (String) e.getValue(), ftTextNotStored));
+ doc.add(new Field(field + "_" + lang, value, ftTextNotStored));
// add fields for any defined auxiliary indexes
List<String> auxIndexes = Util.getAuxIndexes(lang);
if (auxIndexes != null) {
for (String auxTag : auxIndexes) {
- doc.add(new Field(e.getKey() + "_" + auxTag, (String) e.getValue(), ftTextNotStored));
+ doc.add(new Field(field + "_" + auxTag, value, ftTextNotStored));
}
}
}
@@ -366,7 +374,7 @@ public class TextIndexLucene implements TextIndex {
}
}
if (uidField != null) {
- String hash = entity.getChecksum(e.getKey(), (String) e.getValue());
+ String hash = entity.getChecksum(field, value);
doc.add(new Field(uidField, hash, StringField.TYPE_STORED));
}
}
@@ -630,32 +638,25 @@ public class TextIndexLucene implements TextIndex {
} else {
if (this.isMultilingual && StringUtils.isNotEmpty(lang) && !lang.equals("none")) {
textField += "_" + lang;
- }
-
- if (docDef.getField(property) != null) {
+ textClause = textField + ":" + qs;
+ } else if (docDef.getField(property) != null) {
textClause = textField + ":" + qs;
} else {
textClause = qs;
}
-
- String langClause = null;
- if (langField != null) {
- langClause = StringUtils.isNotEmpty(lang) ? (!lang.equals("none") ? langField + ":" + lang : "-" + langField + ":*") : null;
+
+ if (langField != null && StringUtils.isNotEmpty(lang)) {
+ textClause = "(" + textClause + ") AND " + (!lang.equals("none") ? langField + ":" + lang : "-" + langField + ":*");
}
- if (langClause != null)
- textClause = "(" + textClause + ") AND " + langClause ;
}
- String graphClause = null;
- if (graphURI != null) {
- String escaped = QueryParserBase.escape(graphURI) ;
- graphClause = getDocDef().getGraphField() + ":" + escaped ;
- }
String queryString = textClause ;
- if (graphClause != null)
- queryString = "(" + queryString + ") AND " + graphClause ;
+ if (graphURI != null) {
+ String escaped = QueryParserBase.escape(graphURI) ;
+ queryString = "(" + queryString + ") AND " + getDocDef().getGraphField() + ":" + escaped ;
+ }
Analyzer qa = getQueryAnalyzer(usingSearchFor, lang);
Query query = parseQuery(queryString, qa) ;
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java
----------------------------------------------------------------------
diff --git a/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java b/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java
index 48bd94d..c66b0a8 100644
--- a/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java
+++ b/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java
@@ -99,6 +99,7 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble
Multimap<String, Node> mapDefs = HashMultimap.create() ;
Map<String, Analyzer> analyzerDefs = new HashMap<>();
+ Map<String, Boolean> noIndexDefs = new HashMap<>();
Statement listStmt = root.getProperty(TextVocab.pMap);
while (listStmt != null) {
@@ -141,6 +142,16 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble
}
mapDefs.put(field, n.asNode()) ;
+ Statement noIndexStatement = listEntry.getProperty(TextVocab.pNoIndex);
+ if (noIndexStatement != null) {
+ n = noIndexStatement.getObject();
+ if (! n.isLiteral()) {
+ throw new TextIndexException("Text map entry noIndex property must be a boolean : " + n);
+ }
+ boolean noInx = n.asLiteral().getBoolean();
+ noIndexDefs.put(field, noInx) ;
+ }
+
Statement analyzerStatement = listEntry.getProperty(TextVocab.pAnalyzer);
if (analyzerStatement != null) {
n = analyzerStatement.getObject();
@@ -171,6 +182,9 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble
for ( Node p : mapDefs.get(f))
docDef.set(f, p) ;
}
+ for (String f : noIndexDefs.keySet()) {
+ docDef.setNoIndex(f, noIndexDefs.get(f));
+ }
for (String f : analyzerDefs.keySet()) {
docDef.setAnalyzer(f, analyzerDefs.get(f));
}
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java
----------------------------------------------------------------------
diff --git a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java
index ee2bdb3..76f5ac9 100644
--- a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java
+++ b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java
@@ -57,6 +57,7 @@ public class TextVocab
public static final Property pMap = Vocab.property(NS, "map") ;
public static final Property pField = Vocab.property(NS, "field") ;
public static final Property pPredicate = Vocab.property(NS, "predicate") ;
+ public static final Property pNoIndex = Vocab.property(NS, "noIndex") ;
public static final Property pOptional = Vocab.property(NS, "optional") ;
// Analyzers
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/test/java/org/apache/jena/query/text/TestTextMultilingualEnhancements.java
----------------------------------------------------------------------
diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestTextMultilingualEnhancements.java b/jena-text/src/test/java/org/apache/jena/query/text/TestTextMultilingualEnhancements.java
index 5ca5a16..1ac5fff 100644
--- a/jena-text/src/test/java/org/apache/jena/query/text/TestTextMultilingualEnhancements.java
+++ b/jena-text/src/test/java/org/apache/jena/query/text/TestTextMultilingualEnhancements.java
@@ -129,7 +129,7 @@ public class TestTextMultilingualEnhancements extends AbstractTestDatasetWithTex
" text:langField \"lang\" ;",
" text:graphField \"graph\" ;",
" text:map (",
- " [ text:field \"label\" ; text:predicate rdfs:label ]",
+ " [ text:field \"label\" ; text:predicate rdfs:label ; text:noIndex true ]",
" [ text:field \"comment\" ; text:predicate rdfs:comment ]",
" ) ."
);
http://git-wip-us.apache.org/repos/asf/jena/blob/b757303a/jena-text/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/jena-text/src/test/resources/log4j.properties b/jena-text/src/test/resources/log4j.properties
index cf96ece..188b1ca 100644
--- a/jena-text/src/test/resources/log4j.properties
+++ b/jena-text/src/test/resources/log4j.properties
@@ -8,3 +8,6 @@ log4j.appender.stdlog.layout.ConversionPattern=%d{HH:mm:ss} %-5p %-25c{1} :: %m%
# Execution logging
log4j.logger.org.apache.jena.arq.info=INFO
log4j.logger.org.apache.jena.arq.exec=INFO
+
+# XXX: uncomment to get the Lucene queries
+# log4j.logger.org.apache.jena.query.text=TRACE
[2/2] jena git commit: JENA-1646 Merge commit 'refs/pull/504/head' of
https://github.com/apache/jena. This close #504
Posted by co...@apache.org.
JENA-1646 Merge commit 'refs/pull/504/head' of https://github.com/apache/jena. This close #504
Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/31995c78
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/31995c78
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/31995c78
Branch: refs/heads/master
Commit: 31995c78a06b2f9bb1e1760866806e9eaae61307
Parents: 0efda76 b757303
Author: Code Ferret <co...@apache.org>
Authored: Wed Dec 5 12:21:09 2018 -0600
Committer: Code Ferret <co...@apache.org>
Committed: Wed Dec 5 12:21:09 2018 -0600
----------------------------------------------------------------------
.../jena/query/text/EntityDefinition.java | 10 +++++
.../apache/jena/query/text/TextIndexLucene.java | 41 ++++++++++----------
.../assembler/EntityDefinitionAssembler.java | 14 +++++++
.../jena/query/text/assembler/TextVocab.java | 1 +
.../text/TestTextMultilingualEnhancements.java | 2 +-
jena-text/src/test/resources/log4j.properties | 3 ++
6 files changed, 50 insertions(+), 21 deletions(-)
----------------------------------------------------------------------