You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2012/07/30 19:08:59 UTC
svn commit: r1367178 - in /lucene/dev/trunk:
lucene/analysis/common/src/java/org/apache/lucene/analysis/path/
solr/core/src/test-files/solr/collection1/conf/
solr/core/src/test/org/apache/solr/analysis/
solr/example/solr/collection1/conf/
Author: hossman
Date: Mon Jul 30 17:08:58 2012
New Revision: 1367178
URL: http://svn.apache.org/viewvc?rev=1367178&view=rev
Log:
SOLR-3674: better docs, example, test for PathHierarchyTokenizerFactory
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java (with props)
Modified:
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java?rev=1367178&r1=1367177&r2=1367178&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java Mon Jul 30 17:08:58 2012
@@ -27,13 +27,47 @@ import org.apache.lucene.analysis.util.T
/**
* Factory for {@link PathHierarchyTokenizer}.
+ * <p>
+ * This factory is typically configured for use only in the <code>index</code>
+ * Analyzer (or only in the <code>query</code> Analyzer, but never both).
+ * </p>
+ * <p>
+ * For example, in the configuration below a query for
+ * <code>Books/NonFic</code> will match documents indexed with values like
+ * <code>Books/NonFic</code>, <code>Books/NonFic/Law</code>,
+ * <code>Books/NonFic/Science/Physics</code>, etc. But it will not match
+ * documents indexed with values like <code>Books</code>, or
+ * <code>Books/Fic</code>...
+ * </p>
+ *
* <pre class="prettyprint" >
- * <fieldType name="text_path" class="solr.TextField" positionIncrementGap="100">
- * <analyzer>
- * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="\" replace="/"/>
+ * <fieldType name="descendent_path" class="solr.TextField">
+ * <analyzer type="index">
+ * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
* </analyzer>
- * </fieldType></pre>
- *
+ * <analyzer type="query">
+ * <tokenizer class="solr.KeywordTokenizerFactory" />
+ * </analyzer>
+ * </fieldType>
+ * </pre>
+ * <p>
+ * In this example however we see the oposite configuration, so that a query
+ * for <code>Books/NonFic/Science/Physics</code> would match documents
+ * containing <code>Books/NonFic</code>, <code>Books/NonFic/Science</code>,
+ * or <code>Books/NonFic/Science/Physics</code>, but not
+ * <code>Books/NonFic/Science/Physics/Theory</code> or
+ * <code>Books/NonFic/Law</code>.
+ * </p>
+ * <pre class="prettyprint" >
+ * <fieldType name="descendent_path" class="solr.TextField">
+ * <analyzer type="index">
+ * <tokenizer class="solr.KeywordTokenizerFactory" />
+ * </analyzer>
+ * <analyzer type="query">
+ * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ * </analyzer>
+ * </fieldType>
+ * </pre>
*/
public class PathHierarchyTokenizerFactory extends TokenizerFactory {
Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml?rev=1367178&r1=1367177&r2=1367178&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml Mon Jul 30 17:08:58 2012
@@ -414,7 +414,32 @@
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
- </types>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at index time, so
+ queries for paths match documents at that path, or in descendent paths
+ -->
+ <fieldType name="path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at query time, so
+ queries for paths match documents at that path, or in ancestor paths
+ -->
+ <fieldType name="ancestor_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ </fieldType>
+
+</types>
<fields>
@@ -640,6 +665,10 @@
<!-- Type used to index the lat and lon components for the "location" FieldType -->
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" omitNorms="true" />
+
+ <dynamicField name="*_path" type="path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+ <dynamicField name="*_ancestor" type="ancestor_path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+
</fields>
<defaultSearchField>text</defaultSearchField>
@@ -673,6 +702,7 @@
<!-- dynamic destination -->
<copyField source="*_dynamic" dest="dynamic_*"/>
+ <copyField source="*_path" dest="*_ancestor"/>
<!-- example of a custom similarity -->
<similarity class="solr.CustomSimilarityFactory">
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java?rev=1367178&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/PathHierarchyTokenizerFactoryTest.java Mon Jul 30 17:08:58 2012
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.analysis;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+import org.junit.BeforeClass;
+
+public class PathHierarchyTokenizerFactoryTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ initCore("solrconfig.xml","schema.xml");
+
+ assertU(adoc("id", "11",
+ "cat_path", "Movies/Fic/War"));
+
+ assertU(adoc("id", "31",
+ "cat_path", "Books/Fic"));
+ assertU(adoc("id", "31",
+ "cat_path", "Books/Fic/Law"));
+ assertU(adoc("id", "32",
+ "cat_path", "Books/Fic/Science"));
+
+ assertU(adoc("id", "40",
+ "cat_path", "Books/NonFic"));
+ assertU(adoc("id", "41",
+ "cat_path", "Books/NonFic/Law"));
+ assertU(adoc("id", "42",
+ "cat_path", "Books/NonFic/Law",
+ "cat_path", "Books/NonFic/Science"));
+ assertU(adoc("id", "43",
+ "cat_path", "Books/NonFic/Science/Physics",
+ "cat_path", "Books/NonFic/History"));
+
+ assertU(commit());
+ }
+
+ public void testDescendents() throws Exception {
+
+ assertQ(req("{!field f=cat_path}Books/NonFic")
+ ,"//*[@numFound='4']"
+ ,"//int[@name='id' and .='40']"
+ ,"//int[@name='id' and .='41']"
+ ,"//int[@name='id' and .='42']"
+ ,"//int[@name='id' and .='43']"
+ );
+ assertQ(req("{!field f=cat_path}Books/NonFic/Law")
+ ,"//*[@numFound='2']"
+ ,"//int[@name='id' and .='41']"
+ ,"//int[@name='id' and .='42']"
+ );
+
+ assertQ(req("{!field f=cat_path}Books/NonFic/Science")
+ ,"//*[@numFound='2']"
+ ,"//int[@name='id' and .='42']"
+ ,"//int[@name='id' and .='43']"
+ );
+ }
+
+ public void testAncestors() throws Exception {
+
+ assertQ(req("{!field f=cat_ancestor}Books/NonFic/Science")
+ ,"//*[@numFound='2']"
+ ,"//int[@name='id' and .='40']"
+ ,"//int[@name='id' and .='42']"
+ );
+ assertQ(req("{!field f=cat_ancestor}Books/NonFic/Law")
+ ,"//*[@numFound='3']"
+ ,"//int[@name='id' and .='40']"
+ ,"//int[@name='id' and .='41']"
+ ,"//int[@name='id' and .='42']"
+ );
+
+ assertQ(req("{!field f=cat_ancestor}Books/NonFic/Science/Physics")
+ ,"//*[@numFound='3']"
+ ,"//int[@name='id' and .='40']"
+ ,"//int[@name='id' and .='42']"
+ ,"//int[@name='id' and .='43']"
+ );
+ }
+}
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml?rev=1367178&r1=1367177&r2=1367178&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml Mon Jul 30 17:08:58 2012
@@ -137,7 +137,6 @@
<field name="payloads" type="payloads" indexed="true" stored="true"/>
-
<field name="_version_" type="long" indexed="true" stored="true"/>
<!-- Uncommenting the following will create a "timestamp" field using
@@ -607,12 +606,30 @@
</analyzer>
</fieldType>
- <fieldType name="text_path" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.PathHierarchyTokenizerFactory"/>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at index time, so
+ queries for paths match documents at that path, or in descendent paths
+ -->
+ <fieldType name="descendent_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at query time, so
+ queries for paths match documents at that path, or in ancestor paths
+ -->
+ <fieldType name="ancestor_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
</analyzer>
</fieldType>
-
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
@@ -1041,6 +1058,7 @@
<filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
</analyzer>
</fieldType>
+
</types>
<!-- Similarity is the scoring routine for each document vs. a query.