You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2018/05/28 14:52:09 UTC

[1/5] lucene-solr:master: Add 6.6.4 back compat test indexes.

Repository: lucene-solr
Updated Branches:
  refs/heads/branch_7x 9a560563d -> 3b6ab0b34
  refs/heads/master f8ae14405 -> 1971ef310


Add 6.6.4 back compat test indexes.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/78ca82e6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/78ca82e6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/78ca82e6

Branch: refs/heads/master
Commit: 78ca82e63a311bf4eddea6cec8c88ffc5d506110
Parents: f8ae144
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:11:32 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:11:32 2018 +0200

----------------------------------------------------------------------
 .../lucene/index/TestBackwardsCompatibility.java   |   4 +++-
 .../apache/lucene/index/unsupported.6.6.4-cfs.zip  | Bin 0 -> 15782 bytes
 .../lucene/index/unsupported.6.6.4-nocfs.zip       | Bin 0 -> 15771 bytes
 3 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 8e24ea8..da6c9ac 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -483,7 +483,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       "6.6.2-cfs",
       "6.6.2-nocfs",
       "6.6.3-cfs",
-      "6.6.3-nocfs"
+      "6.6.3-nocfs",
+      "6.6.4-cfs",
+      "6.6.4-nocfs"
   };
 
   // TODO: on 6.0.0 release, gen the single segment indices and add here:

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip
new file mode 100644
index 0000000..9a256d8
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip
new file mode 100644
index 0000000..a25dbef
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip differ


[3/5] lucene-solr:branch_7x: LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.

Posted by jp...@apache.org.
LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ea8927f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ea8927f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ea8927f

Branch: refs/heads/branch_7x
Commit: 9ea8927f1c33edcd041b5b83f22af0e0a473ed54
Parents: 9a56056
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:20:49 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:21:12 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                                      |  3 +++
 .../apache/lucene/analysis/custom/CustomAnalyzer.java   |  6 ++++++
 .../lucene/analysis/custom/TestCustomAnalyzer.java      | 12 ++++++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0fa942c..3c798e0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -144,6 +144,9 @@ Bug Fixes
 * LUCENE-8325: Fixed the smartcn tokenizer to not split UTF-16 surrogate pairs.
   (chengpohi via Jim Ferenczi)
 
+* LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term
+  queries. (Tim Allison via Adrien Grand)
+
 Other
 
 * LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
index 19e207f..f60c6a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
@@ -162,6 +162,12 @@ public final class CustomAnalyzer extends Analyzer {
   @Override
   protected TokenStream normalize(String fieldName, TokenStream in) {
     TokenStream result = in;
+    // tokenizers can return a tokenfilter if the tokenizer does normalization,
+    // although this is really bogus/abstraction violation...
+    if (tokenizer instanceof MultiTermAwareComponent) {
+      TokenFilterFactory filter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenizer).getMultiTermComponent();
+      result = filter.create(result);
+    }
     for (TokenFilterFactory filter : tokenFilters) {
       if (filter instanceof MultiTermAwareComponent) {
         filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index 7583884..5c36e82 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -31,9 +31,9 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
 import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
-import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
 import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
 import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseTokenizerFactory;
 import org.apache.lucene.analysis.core.StopFilterFactory;
 import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
@@ -432,7 +432,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
 
     @Override
     public AbstractAnalysisFactory getMultiTermComponent() {
-      return new KeywordTokenizerFactory(getOriginalArgs());
+      return new DummyTokenFilterFactory(Collections.emptyMap());
     }
     
   }
@@ -500,6 +500,14 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
         .build();
     assertEquals(new BytesRef("e f c"), analyzer.normalize("dummy", "a b c"));
   }
+  
+  /** test normalize where the TokenizerFactory returns a filter to normalize the text */
+  public void testNormalizationWithLowerCaseTokenizer() throws IOException {
+    CustomAnalyzer analyzer1 = CustomAnalyzer.builder()
+        .withTokenizer(LowerCaseTokenizerFactory.class, Collections.emptyMap())
+        .build();
+    assertEquals(new BytesRef("abc"), analyzer1.normalize("dummy", "ABC"));
+  }
 
   public void testConditions() throws IOException {
     CustomAnalyzer analyzer = CustomAnalyzer.builder()


[4/5] lucene-solr:branch_7x: Add the 6.6.4 version.

Posted by jp...@apache.org.
Add the 6.6.4 version.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/51617c2b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/51617c2b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/51617c2b

Branch: refs/heads/branch_7x
Commit: 51617c2b969da1ea16effdf95e849e398380becf
Parents: 9ea8927
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:35:35 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:35:35 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                                       | 3 +++
 lucene/core/src/java/org/apache/lucene/util/Version.java | 7 +++++++
 2 files changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51617c2b/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3c798e0..f331601 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -840,6 +840,9 @@ Other
   that are trivially replaced by LeafReader.terms() and MultiFields.getTerms()
   (David Smiley)
 
+======================= Lucene 6.6.4 =======================
+(No Changes)
+
 ======================= Lucene 6.6.3 =======================
 
 Build

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51617c2b/lucene/core/src/java/org/apache/lucene/util/Version.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index 785c42c..418c9ff 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -137,6 +137,13 @@ public final class Version {
   public static final Version LUCENE_6_6_3 = new Version(6, 6, 3);
 
   /**
+   * Match settings and bugs in Lucene's 6.6.4 release.
+   * @deprecated Use latest
+   */
+  @Deprecated
+  public static final Version LUCENE_6_6_4 = new Version(6, 6, 4);
+
+  /**
    * Match settings and bugs in Lucene's 7.0.0 release.
    * @deprecated (7.1.0) Use latest
    */


[2/5] lucene-solr:master: LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.

Posted by jp...@apache.org.
LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1971ef31
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1971ef31
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1971ef31

Branch: refs/heads/master
Commit: 1971ef310906239d88602444ae6b74081648f3e4
Parents: 78ca82e
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:20:49 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:20:49 2018 +0200

----------------------------------------------------------------------
 lucene/CHANGES.txt                                      |  3 +++
 .../apache/lucene/analysis/custom/CustomAnalyzer.java   |  6 ++++++
 .../lucene/analysis/custom/TestCustomAnalyzer.java      | 12 ++++++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b643af1..d3f89ba 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -248,6 +248,9 @@ Bug Fixes
 * LUCENE-8325: Fixed the smartcn tokenizer to not split UTF-16 surrogate pairs.
   (chengpohi via Jim Ferenczi)
 
+* LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term
+  queries. (Tim Allison via Adrien Grand)
+
 Other
 
 * LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
index 19e207f..f60c6a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
@@ -162,6 +162,12 @@ public final class CustomAnalyzer extends Analyzer {
   @Override
   protected TokenStream normalize(String fieldName, TokenStream in) {
     TokenStream result = in;
+    // tokenizers can return a tokenfilter if the tokenizer does normalization,
+    // although this is really bogus/abstraction violation...
+    if (tokenizer instanceof MultiTermAwareComponent) {
+      TokenFilterFactory filter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenizer).getMultiTermComponent();
+      result = filter.create(result);
+    }
     for (TokenFilterFactory filter : tokenFilters) {
       if (filter instanceof MultiTermAwareComponent) {
         filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index 01ad75c..1fa59d1 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -31,9 +31,9 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
 import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
-import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
 import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
 import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseTokenizerFactory;
 import org.apache.lucene.analysis.core.StopFilterFactory;
 import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
@@ -432,7 +432,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
 
     @Override
     public AbstractAnalysisFactory getMultiTermComponent() {
-      return new KeywordTokenizerFactory(getOriginalArgs());
+      return new DummyTokenFilterFactory(Collections.emptyMap());
     }
     
   }
@@ -500,6 +500,14 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
         .build();
     assertEquals(new BytesRef("e f c"), analyzer.normalize("dummy", "a b c"));
   }
+  
+  /** test normalize where the TokenizerFactory returns a filter to normalize the text */
+  public void testNormalizationWithLowerCaseTokenizer() throws IOException {
+    CustomAnalyzer analyzer1 = CustomAnalyzer.builder()
+        .withTokenizer(LowerCaseTokenizerFactory.class, Collections.emptyMap())
+        .build();
+    assertEquals(new BytesRef("abc"), analyzer1.normalize("dummy", "ABC"));
+  }
 
   public void testConditions() throws IOException {
     CustomAnalyzer analyzer = CustomAnalyzer.builder()


[5/5] lucene-solr:branch_7x: Add 6.6.4 back compat test indexes

Posted by jp...@apache.org.
Add 6.6.4 back compat test indexes


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3b6ab0b3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3b6ab0b3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3b6ab0b3

Branch: refs/heads/branch_7x
Commit: 3b6ab0b34b4784fe1ebb28e3cf6ebbd1786cca37
Parents: 51617c2
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:36:12 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:36:12 2018 +0200

----------------------------------------------------------------------
 .../lucene/index/TestBackwardsCompatibility.java  |   3 +++
 .../org/apache/lucene/index/index.6.6.4-cfs.zip   | Bin 0 -> 15776 bytes
 .../org/apache/lucene/index/index.6.6.4-nocfs.zip | Bin 0 -> 15768 bytes
 .../test/org/apache/lucene/index/sorted.6.6.4.zip | Bin 0 -> 156997 bytes
 4 files changed, 3 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 8fc9373..b7adc5a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -317,6 +317,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     "6.6.2-nocfs",
     "6.6.3-cfs",
     "6.6.3-nocfs",
+    "6.6.4-cfs",
+    "6.6.4-nocfs",
     "7.0.0-cfs",
     "7.0.0-nocfs",
     "7.0.1-cfs",
@@ -350,6 +352,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     "sorted.6.6.1",
     "sorted.6.6.2",
     "sorted.6.6.3",
+    "sorted.6.6.4",
     "sorted.7.0.0",
     "sorted.7.0.1",
     "sorted.7.1.0",

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip
new file mode 100644
index 0000000..226949b
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip
new file mode 100644
index 0000000..c78ddb6
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip
new file mode 100644
index 0000000..829de0b
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip differ