You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/03/19 19:22:29 UTC
svn commit: r1458442 - in /lucene/dev/branches/lucene_solr_4_2: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/classification/ lucene/classification/src/ lucen...
Author: markrmiller
Date: Tue Mar 19 18:22:27 2013
New Revision: 1458442
URL: http://svn.apache.org/r1458442
Log:
SOLR-4538: Date Math expressions were being truncated to 32 characters when used in field:value queries in the lucene QParser.
Modified:
lucene/dev/branches/lucene_solr_4_2/ (props changed)
lucene/dev/branches/lucene_solr_4_2/dev-tools/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/BUILD.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/CHANGES.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/LICENSE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/README.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/analysis/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/backwards/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/benchmark/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/src/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/codecs/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/demo/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/facet/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/grouping/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/join/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/licenses/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/memory/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/misc/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/module-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/queries/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/site/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/spatial/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/suggest/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/tools/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_2/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/README.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/cloud-dev/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
lucene/dev/branches/lucene_solr_4_2/solr/example/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/scripts/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/site/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/testlogging.properties (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/webapp/ (props changed)
Modified: lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt?rev=1458442&r1=1458441&r2=1458442&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt Tue Mar 19 18:22:27 2013
@@ -120,6 +120,9 @@ Bug Fixes
handler-level parameters were specified containing periods in the name
(James Dyer)
+* SOLR-4538: Date Math expressions were being truncated to 32 characters
+ when used in field:value queries in the lucene QParser. (hossman, yonik)
+
Optimizations
----------------------
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1458442&r1=1458441&r2=1458442&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Tue Mar 19 18:22:27 2013
@@ -17,6 +17,8 @@
package org.apache.solr.analysis;
import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -59,10 +61,11 @@ final class TrieTokenizer extends Tokeni
protected final TrieTypes type;
protected final NumericTokenStream ts;
+ // NumericTokenStream does not support CharTermAttribute so keep it local
+ private final CharTermAttribute termAtt = new CharTermAttributeImpl();
protected final OffsetAttribute ofsAtt = addAttribute(OffsetAttribute.class);
protected int startOfs, endOfs;
protected boolean hasValue;
- protected final char[] buf = new char[32];
static NumericTokenStream getNumericTokenStream(int precisionStep) {
return new NumericTokenStream(precisionStep);
@@ -73,22 +76,23 @@ final class TrieTokenizer extends Tokeni
super(ts, input);
this.type = type;
this.ts = ts;
+ // dates tend to be longer, especially when math is involved
+ termAtt.resizeBuffer( type == TrieTypes.DATE ? 128 : 32 );
}
@Override
public void reset() {
try {
int upto = 0;
- while (upto < buf.length) {
- final int length = input.read(buf, upto, buf.length - upto);
+ char[] buf = termAtt.buffer();
+ while (true) {
+ final int length = input.read(buf, upto, buf.length-upto);
if (length == -1) break;
upto += length;
+ if (upto == buf.length)
+ buf = termAtt.resizeBuffer(1+buf.length);
}
- // skip remaining data if buffer was too short:
- if (upto == buf.length) {
- input.skip(Long.MAX_VALUE);
- }
-
+ termAtt.setLength(upto);
this.startOfs = correctOffset(0);
this.endOfs = correctOffset(upto);
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1458442&r1=1458441&r2=1458442&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Tue Mar 19 18:22:27 2013
@@ -784,7 +784,35 @@ public class BasicFunctionalityTest exte
assertU(adoc("id", "5", "bday", "NOW+30MINUTES"));
assertU(adoc("id", "6", "bday", "NOW+2YEARS"));
assertU(commit());
+
+ // a ridiculoulsy long date math expression that's still equivilent to july4
+ final StringBuilder july4Long = new StringBuilder(july4);
+ final int iters = atLeast(10);
+ for (int i = 0; i < iters; i++) {
+ final String val = String.valueOf(atLeast(1));
+ july4Long.append("+" + val + "SECONDS-" + val + "SECONDS");
+ }
+
+ // term queries using date math (all of these should match doc#1)
+ for (String q :
+ new String[] {
+ "bday:1976-07-04T12\\:08\\:56.45Z/SECOND+235MILLIS",
+ "bday:1976-07-04T12\\:08\\:56.123Z/MINUTE+56SECONDS+235MILLIS",
+ "bday:\"1976-07-04T12:08:56.45Z/SECOND+235MILLIS\"",
+ "bday:\"1976-07-04T12:08:56.123Z/MINUTE+56SECONDS+235MILLIS\"",
+ "{!term f=bday}1976-07-04T12:08:56.45Z/SECOND+235MILLIS",
+ "{!term f=bday}1976-07-04T12:08:56.123Z/MINUTE+56SECONDS+235MILLIS",
+ "{!term f=bday}"+july4,
+ "{!term f=bday}"+july4Long,
+ "bday:\"" + july4Long + "\""
+ }) {
+ assertQ("check math on field query: " + q,
+ req("q", q),
+ "*[count(//doc)=1]",
+ "//int[@name='id'][.='1']");
+ }
+ // range queries using date math
assertQ("check math on absolute date#1",
req("q", "bday:[* TO "+july4+"/SECOND]"),
"*[count(//doc)=0]");
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1458442&r1=1458441&r2=1458442&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Tue Mar 19 18:22:27 2013
@@ -75,7 +75,20 @@ public class QueryEqualityTest extends S
private static final Set<String> valParsersTested = new HashSet<String>();
+ public void testDateMathParsingEquality() throws Exception {
+ // regardless of parser, these should all be equivilent queries
+ assertQueryEquals
+ (null
+ ,"{!lucene}f_tdt:2013-09-11T00\\:00\\:00Z"
+ ,"{!lucene}f_tdt:2013-03-08T00\\:46\\:15Z/DAY+6MONTHS+3DAYS"
+ ,"{!lucene}f_tdt:\"2013-03-08T00:46:15Z/DAY+6MONTHS+3DAYS\""
+ ,"{!field f=f_tdt}2013-03-08T00:46:15Z/DAY+6MONTHS+3DAYS"
+ ,"{!field f=f_tdt}2013-09-11T00:00:00Z"
+ ,"{!term f=f_tdt}2013-03-08T00:46:15Z/DAY+6MONTHS+3DAYS"
+ ,"{!term f=f_tdt}2013-09-11T00:00:00Z"
+ );
+ }
public void testQueryLucene() throws Exception {
assertQueryEquals("lucene", "{!lucene}apache solr",
"apache solr", "apache solr ");
@@ -720,8 +733,8 @@ public class QueryEqualityTest extends S
}
/**
- * NOTE: defType is not only used to pick the parser, but also to record
- * the parser being tested for coverage sanity checking
+ * NOTE: defType is not only used to pick the parser, but, if non-null it is
+ * also to record the parser being tested for coverage sanity checking
*
* @see QueryUtils#check
* @see QueryUtils#checkEqual
@@ -730,7 +743,8 @@ public class QueryEqualityTest extends S
protected void assertQueryEquals(final String defType,
final SolrQueryRequest req,
final String... inputs) throws Exception {
- qParsersTested.add(defType);
+
+ if (null != defType) qParsersTested.add(defType);
final Query[] queries = new Query[inputs.length];
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java?rev=1458442&r1=1458441&r2=1458442&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java Tue Mar 19 18:22:27 2013
@@ -104,6 +104,11 @@ public class TestSolrQueryParser extends
);
assertJQ(req("q","doesnotexist_s:( * * * )")
,"/response/numFound==0" // nothing should be found
+ );
+
+ // length of date math caused issues...
+ assertJQ(req("q","foo_dt:\"2013-03-08T00:46:15Z/DAY+000MILLISECONDS+00SECONDS+00MINUTES+00HOURS+0000000000YEARS+6MONTHS+3DAYS\"", "debug","query")
+ ,"/debug/parsedquery=='foo_dt:2013-09-11T00:00:00Z'"
);
}