You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2014/09/17 15:45:08 UTC
svn commit: r1625586 - in
/lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common: build.xml
src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
Author: sarowe
Date: Wed Sep 17 13:45:08 2014
New Revision: 1625586
URL: http://svn.apache.org/r1625586
Log:
LUCENE-5897, LUCENE-5400: change JFlex-generated source munging so that zzRefill() doesn't call Reader.read(buffer,start,len) with len=0
Modified:
lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/build.xml
lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
Modified: lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/build.xml?rev=1625586&r1=1625585&r2=1625586&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/build.xml (original)
+++ lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/build.xml Wed Sep 17 13:45:08 2014
@@ -104,8 +104,11 @@
match="private static final int ZZ_BUFFERSIZE ="
replace="private int ZZ_BUFFERSIZE ="/>
<replaceregexp file="@{dir}/@{name}.java"
+ match="int numRead = zzReader.read\(zzBuffer, zzEndRead,\s+zzBuffer.length-zzEndRead\);"
+ replace="int numRead = 0, requested = zzBuffer.length - zzEndRead;
 if (requested > 0) numRead = zzReader.read(zzBuffer, zzEndRead, requested);"/>
+ <replaceregexp file="@{dir}/@{name}.java"
match="(// unlikely but not impossible:.*?if \(numRead == 0)"
- replace="\1 && zzBuffer.length - zzEndRead > 0"
+ replace="\1 && requested > 0"
flags="s" />
</sequential>
</macrodef>
Modified: lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1625586&r1=1625585&r2=1625586&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Wed Sep 17 13:45:08 2014
@@ -1023,15 +1023,15 @@ public final class StandardTokenizerImpl
/* finally: fill the buffer with new input */
- int numRead = zzReader.read(zzBuffer, zzEndRead,
- zzBuffer.length-zzEndRead);
+ int numRead = 0, requested = zzBuffer.length - zzEndRead;
+ if (requested > 0) numRead = zzReader.read(zzBuffer, zzEndRead, requested);
if (numRead > 0) {
zzEndRead+= numRead;
return false;
}
// unlikely but not impossible: read 0 characters, but not at end of stream
- if (numRead == 0 && zzBuffer.length - zzEndRead > 0) {
+ if (numRead == 0 && requested > 0) {
int c = zzReader.read();
if (c == -1) {
return true;
Modified: lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1625586&r1=1625585&r2=1625586&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene_solr_4_9/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java Wed Sep 17 13:45:08 2014
@@ -9200,15 +9200,15 @@ public final class UAX29URLEmailTokenize
/* finally: fill the buffer with new input */
- int numRead = zzReader.read(zzBuffer, zzEndRead,
- zzBuffer.length-zzEndRead);
+ int numRead = 0, requested = zzBuffer.length - zzEndRead;
+ if (requested > 0) numRead = zzReader.read(zzBuffer, zzEndRead, requested);
if (numRead > 0) {
zzEndRead+= numRead;
return false;
}
// unlikely but not impossible: read 0 characters, but not at end of stream
- if (numRead == 0 && zzBuffer.length - zzEndRead > 0) {
+ if (numRead == 0 && requested > 0) {
int c = zzReader.read();
if (c == -1) {
return true;