You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2018/05/28 14:52:09 UTC
[1/5] lucene-solr:master: Add 6.6.4 back compat test indexes.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 9a560563d -> 3b6ab0b34
refs/heads/master f8ae14405 -> 1971ef310
Add 6.6.4 back compat test indexes.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/78ca82e6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/78ca82e6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/78ca82e6
Branch: refs/heads/master
Commit: 78ca82e63a311bf4eddea6cec8c88ffc5d506110
Parents: f8ae144
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:11:32 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:11:32 2018 +0200
----------------------------------------------------------------------
.../lucene/index/TestBackwardsCompatibility.java | 4 +++-
.../apache/lucene/index/unsupported.6.6.4-cfs.zip | Bin 0 -> 15782 bytes
.../lucene/index/unsupported.6.6.4-nocfs.zip | Bin 0 -> 15771 bytes
3 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 8e24ea8..da6c9ac 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -483,7 +483,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"6.6.2-cfs",
"6.6.2-nocfs",
"6.6.3-cfs",
- "6.6.3-nocfs"
+ "6.6.3-nocfs",
+ "6.6.4-cfs",
+ "6.6.4-nocfs"
};
// TODO: on 6.0.0 release, gen the single segment indices and add here:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip
new file mode 100644
index 0000000..9a256d8
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/78ca82e6/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip
new file mode 100644
index 0000000..a25dbef
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.4-nocfs.zip differ
[3/5] lucene-solr:branch_7x: LUCENE-8186: LowerCaseTokenizerFactory
now lowercases text in multi-term queries.
Posted by jp...@apache.org.
LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ea8927f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ea8927f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ea8927f
Branch: refs/heads/branch_7x
Commit: 9ea8927f1c33edcd041b5b83f22af0e0a473ed54
Parents: 9a56056
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:20:49 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:21:12 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/analysis/custom/CustomAnalyzer.java | 6 ++++++
.../lucene/analysis/custom/TestCustomAnalyzer.java | 12 ++++++++++--
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0fa942c..3c798e0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -144,6 +144,9 @@ Bug Fixes
* LUCENE-8325: Fixed the smartcn tokenizer to not split UTF-16 surrogate pairs.
(chengpohi via Jim Ferenczi)
+* LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term
+ queries. (Tim Allison via Adrien Grand)
+
Other
* LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
index 19e207f..f60c6a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
@@ -162,6 +162,12 @@ public final class CustomAnalyzer extends Analyzer {
@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = in;
+ // tokenizers can return a tokenfilter if the tokenizer does normalization,
+ // although this is really bogus/abstraction violation...
+ if (tokenizer instanceof MultiTermAwareComponent) {
+ TokenFilterFactory filter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenizer).getMultiTermComponent();
+ result = filter.create(result);
+ }
for (TokenFilterFactory filter : tokenFilters) {
if (filter instanceof MultiTermAwareComponent) {
filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ea8927f/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index 7583884..5c36e82 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -31,9 +31,9 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
-import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseTokenizerFactory;
import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
@@ -432,7 +432,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
@Override
public AbstractAnalysisFactory getMultiTermComponent() {
- return new KeywordTokenizerFactory(getOriginalArgs());
+ return new DummyTokenFilterFactory(Collections.emptyMap());
}
}
@@ -500,6 +500,14 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
.build();
assertEquals(new BytesRef("e f c"), analyzer.normalize("dummy", "a b c"));
}
+
+ /** test normalize where the TokenizerFactory returns a filter to normalize the text */
+ public void testNormalizationWithLowerCaseTokenizer() throws IOException {
+ CustomAnalyzer analyzer1 = CustomAnalyzer.builder()
+ .withTokenizer(LowerCaseTokenizerFactory.class, Collections.emptyMap())
+ .build();
+ assertEquals(new BytesRef("abc"), analyzer1.normalize("dummy", "ABC"));
+ }
public void testConditions() throws IOException {
CustomAnalyzer analyzer = CustomAnalyzer.builder()
[4/5] lucene-solr:branch_7x: Add the 6.6.4 version.
Posted by jp...@apache.org.
Add the 6.6.4 version.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/51617c2b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/51617c2b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/51617c2b
Branch: refs/heads/branch_7x
Commit: 51617c2b969da1ea16effdf95e849e398380becf
Parents: 9ea8927
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:35:35 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:35:35 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
lucene/core/src/java/org/apache/lucene/util/Version.java | 7 +++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51617c2b/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3c798e0..f331601 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -840,6 +840,9 @@ Other
that are trivially replaced by LeafReader.terms() and MultiFields.getTerms()
(David Smiley)
+======================= Lucene 6.6.4 =======================
+(No Changes)
+
======================= Lucene 6.6.3 =======================
Build
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51617c2b/lucene/core/src/java/org/apache/lucene/util/Version.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java
index 785c42c..418c9ff 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Version.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Version.java
@@ -137,6 +137,13 @@ public final class Version {
public static final Version LUCENE_6_6_3 = new Version(6, 6, 3);
/**
+ * Match settings and bugs in Lucene's 6.6.4 release.
+ * @deprecated Use latest
+ */
+ @Deprecated
+ public static final Version LUCENE_6_6_4 = new Version(6, 6, 4);
+
+ /**
* Match settings and bugs in Lucene's 7.0.0 release.
* @deprecated (7.1.0) Use latest
*/
[2/5] lucene-solr:master: LUCENE-8186: LowerCaseTokenizerFactory now
lowercases text in multi-term queries.
Posted by jp...@apache.org.
LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term queries.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1971ef31
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1971ef31
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1971ef31
Branch: refs/heads/master
Commit: 1971ef310906239d88602444ae6b74081648f3e4
Parents: 78ca82e
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:20:49 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:20:49 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../apache/lucene/analysis/custom/CustomAnalyzer.java | 6 ++++++
.../lucene/analysis/custom/TestCustomAnalyzer.java | 12 ++++++++++--
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b643af1..d3f89ba 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -248,6 +248,9 @@ Bug Fixes
* LUCENE-8325: Fixed the smartcn tokenizer to not split UTF-16 surrogate pairs.
(chengpohi via Jim Ferenczi)
+* LUCENE-8186: LowerCaseTokenizerFactory now lowercases text in multi-term
+ queries. (Tim Allison via Adrien Grand)
+
Other
* LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
index 19e207f..f60c6a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
@@ -162,6 +162,12 @@ public final class CustomAnalyzer extends Analyzer {
@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = in;
+ // tokenizers can return a tokenfilter if the tokenizer does normalization,
+ // although this is really bogus/abstraction violation...
+ if (tokenizer instanceof MultiTermAwareComponent) {
+ TokenFilterFactory filter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenizer).getMultiTermComponent();
+ result = filter.create(result);
+ }
for (TokenFilterFactory filter : tokenFilters) {
if (filter instanceof MultiTermAwareComponent) {
filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1971ef31/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index 01ad75c..1fa59d1 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -31,9 +31,9 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
-import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseTokenizerFactory;
import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
@@ -432,7 +432,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
@Override
public AbstractAnalysisFactory getMultiTermComponent() {
- return new KeywordTokenizerFactory(getOriginalArgs());
+ return new DummyTokenFilterFactory(Collections.emptyMap());
}
}
@@ -500,6 +500,14 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
.build();
assertEquals(new BytesRef("e f c"), analyzer.normalize("dummy", "a b c"));
}
+
+ /** test normalize where the TokenizerFactory returns a filter to normalize the text */
+ public void testNormalizationWithLowerCaseTokenizer() throws IOException {
+ CustomAnalyzer analyzer1 = CustomAnalyzer.builder()
+ .withTokenizer(LowerCaseTokenizerFactory.class, Collections.emptyMap())
+ .build();
+ assertEquals(new BytesRef("abc"), analyzer1.normalize("dummy", "ABC"));
+ }
public void testConditions() throws IOException {
CustomAnalyzer analyzer = CustomAnalyzer.builder()
[5/5] lucene-solr:branch_7x: Add 6.6.4 back compat test indexes
Posted by jp...@apache.org.
Add 6.6.4 back compat test indexes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3b6ab0b3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3b6ab0b3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3b6ab0b3
Branch: refs/heads/branch_7x
Commit: 3b6ab0b34b4784fe1ebb28e3cf6ebbd1786cca37
Parents: 51617c2
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon May 28 16:36:12 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Mon May 28 16:36:12 2018 +0200
----------------------------------------------------------------------
.../lucene/index/TestBackwardsCompatibility.java | 3 +++
.../org/apache/lucene/index/index.6.6.4-cfs.zip | Bin 0 -> 15776 bytes
.../org/apache/lucene/index/index.6.6.4-nocfs.zip | Bin 0 -> 15768 bytes
.../test/org/apache/lucene/index/sorted.6.6.4.zip | Bin 0 -> 156997 bytes
4 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 8fc9373..b7adc5a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -317,6 +317,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"6.6.2-nocfs",
"6.6.3-cfs",
"6.6.3-nocfs",
+ "6.6.4-cfs",
+ "6.6.4-nocfs",
"7.0.0-cfs",
"7.0.0-nocfs",
"7.0.1-cfs",
@@ -350,6 +352,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"sorted.6.6.1",
"sorted.6.6.2",
"sorted.6.6.3",
+ "sorted.6.6.4",
"sorted.7.0.0",
"sorted.7.0.1",
"sorted.7.1.0",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip
new file mode 100644
index 0000000..226949b
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip
new file mode 100644
index 0000000..c78ddb6
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.6.4-nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b6ab0b3/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip
new file mode 100644
index 0000000..829de0b
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.6.6.4.zip differ