You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@harmony.apache.org by te...@apache.org on 2010/06/15 13:05:28 UTC
svn commit: r954817 - in
/harmony/enhanced/java/trunk/classlib/modules/luni/src:
main/java/java/util/Scanner.java
test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
Author: tellison
Date: Tue Jun 15 11:05:28 2010
New Revision: 954817
URL: http://svn.apache.org/viewvc?rev=954817&view=rev
Log:
Apply patch for HARMONY-6470 ([classlib][luni] Scanner will exhaust all heap memory when parse a large file)
Includes a (relatively) long running test case.
Added:
harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java (with props)
Modified:
harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
Modified: harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
URL: http://svn.apache.org/viewvc/harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java?rev=954817&r1=954816&r2=954817&view=diff
==============================================================================
--- harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java (original)
+++ harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java Tue Jun 15 11:05:28 2010
@@ -2017,6 +2017,8 @@ public final class Scanner implements It
return negative;
}
+ private int tokenStart = 0;
+ private int tokenEnd = 0;
/*
* Find the prefixed delimiter and posefixed delimiter in the input resource
* and set the start index and end index of Matcher region. If postfixed
@@ -2031,11 +2033,13 @@ public final class Scanner implements It
matcher.usePattern(delimiter);
matcher.region(findStartIndex, bufferLength);
- tokenStartIndex = findPreDelimiter();
- if (setHeadTokenRegion(tokenStartIndex)) {
+ findPreDelimiter();
+ if (setHeadTokenRegion(tokenStart)) {
return true;
}
- tokenEndIndex = findPostDelimiter();
+ findPostDelimiter();
+ tokenStartIndex = tokenStart;
+ tokenEndIndex = tokenEnd;
// If the second delimiter is not found
if (-1 == tokenEndIndex) {
// Just first Delimiter Exists
@@ -2053,8 +2057,8 @@ public final class Scanner implements It
/*
* Find prefix delimiter
*/
- private int findPreDelimiter() {
- int tokenStartIndex;
+ private void findPreDelimiter() {
+ tokenStart = 0;
boolean findComplete = false;
while (!findComplete) {
if (matcher.find()) {
@@ -2074,13 +2078,14 @@ public final class Scanner implements It
readMore();
resetMatcher();
} else {
- return -1;
+ tokenStart = -1;
+ return;
}
}
}
- tokenStartIndex = matcher.end();
+
+ tokenStart = matcher.end();
findStartIndex = matcher.end();
- return tokenStartIndex;
}
/*
@@ -2113,8 +2118,8 @@ public final class Scanner implements It
/*
* Find postfix delimiter
*/
- private int findPostDelimiter() {
- int tokenEndIndex = 0;
+ private void findPostDelimiter() {
+ tokenEnd = 0;
boolean findComplete = false;
while (!findComplete) {
if (matcher.find()) {
@@ -2128,13 +2133,16 @@ public final class Scanner implements It
readMore();
resetMatcher();
} else {
- return -1;
+ tokenStart = findStartIndex;
+ tokenEnd = -1;
+ return;
}
}
}
- tokenEndIndex = matcher.start();
+
+ tokenStart = findStartIndex;
+ tokenEnd = matcher.start();
findStartIndex = matcher.start();
- return tokenEndIndex;
}
/*
@@ -2143,13 +2151,12 @@ public final class Scanner implements It
* true, otherwise set to false.
*/
private void readMore() {
- int oldPosition = buffer.position();
- int oldBufferLength = bufferLength;
// Increase capacity if empty space is not enough
if (bufferLength >= buffer.capacity()) {
expandBuffer();
}
-
+ int oldPosition = buffer.position();
+ int oldBufferLength = bufferLength;
// Read input resource
int readCount = 0;
try {
@@ -2183,14 +2190,66 @@ public final class Scanner implements It
// Expand the size of internal buffer.
private void expandBuffer() {
- int oldPosition = buffer.position();
- int oldCapacity = buffer.capacity();
+ //inital new indexes values
+ int nFindStartIndex = 0;
+ int nPreStartIndex = 0;
+ int nBufferLength = 0;
+ int nLimit = 0;
+ int nPostion = 0;
+ int nCachehasNextIndex = 0;
+
+ //store old buffer set
+ int oldLength = bufferLength;
+ int oldPostion = buffer.position();
int oldLimit = buffer.limit();
- int newCapacity = oldCapacity * DIPLOID;
- char[] newBuffer = new char[newCapacity];
- System.arraycopy(buffer.array(), 0, newBuffer, 0, oldLimit);
- buffer = CharBuffer.wrap(newBuffer, 0, newCapacity);
- buffer.position(oldPosition);
- buffer.limit(oldLimit);
+ int oldCapacity = buffer.capacity();
+
+ //new start index for new buffer
+ int startIndex = 0;
+
+ if (preStartIndex == -1) {
+ startIndex = findStartIndex;
+ nPreStartIndex = -1;
+ nFindStartIndex = 0;
+ } else {
+ startIndex = preStartIndex;
+ nPreStartIndex = 0;
+ nFindStartIndex = findStartIndex - startIndex;
+ }
+
+ int nCapacity = oldCapacity;
+ char[] newBuffer = null;
+ // set nCapacity
+ if (startIndex < oldCapacity / 2) {
+ nCapacity = nCapacity << 1;
+ newBuffer = new char[nCapacity];
+ } else {
+ newBuffer = buffer.array();
+ }
+
+ //new buffer set
+ nBufferLength = oldLength - startIndex;
+ nLimit = oldLimit - startIndex;
+ if (oldPostion < startIndex) {
+ nPostion = 0;
+ } else {
+ nPostion = oldPostion - startIndex;
+ }
+ // new CachedhasNextIndex;
+ if (cachehasNextIndex != -1) {
+ nCachehasNextIndex = cachehasNextIndex - startIndex;
+ } else {
+ nCachehasNextIndex = -1;
+ }
+ System.arraycopy(buffer.array(), startIndex, newBuffer, 0, nLimit);
+
+ buffer = CharBuffer.wrap(newBuffer, 0, nCapacity);
+ buffer.position(nPostion);
+ buffer.limit(nLimit);
+
+ preStartIndex = nPreStartIndex;
+ findStartIndex = nFindStartIndex;
+ bufferLength = nBufferLength;
+ cachehasNextIndex = nCachehasNextIndex;
}
}
Added: harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
URL: http://svn.apache.org/viewvc/harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java?rev=954817&view=auto
==============================================================================
--- harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java (added)
+++ harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java Tue Jun 15 11:05:28 2010
@@ -0,0 +1,77 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.harmony.luni.tests.java.util;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Scanner;
+
+import org.junit.Test;
+
+public class ScannerParseLargeFileBenchmarkTest {
+
+ /**
+ * This test will check when parse a large file like more than 200M bytes if
+ * the Scanner will exhaust all heap memory
+ */
+ @Test
+ public void testParseLargeFile() throws Exception {
+ MyReader reader = new MyReader();
+ String delimiter = "\r?\n";
+ Scanner scanner = new Scanner(reader).useDelimiter(delimiter);
+
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ scanner.close();
+ reader.close();
+ }
+
+ private static class MyReader extends Reader {
+ static final char[] CONTENT = "large file!\n".toCharArray();
+
+ static long fileLength = (8 << 21) * 12;
+
+ static boolean first = true;
+
+ static int position = 0;
+
+ private int count = 0;
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public int read(char[] buf, int offset, int length) {
+ if (count >= fileLength) {
+ return -1;
+ }
+ if (first == true) {
+ position = 0;
+ first = false;
+ }
+ for (int i = offset; i < length; i++) {
+ buf[i] = CONTENT[(i + position) % CONTENT.length];
+ count++;
+ }
+
+ position = (length + position) % CONTENT.length;
+
+ return length - offset;
+ }
+ }
+}
Propchange: harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Patch for HARMONY-6470 ([classlib][luni] Scanner will exhaust all
heap memory when parse a large file)
Posted by Tim Ellison <t....@gmail.com>.
I committed this regression test, even though it takes about 1 min to
run on my machine.
If it causes any problems, or if you have a better idea for testing this
case, please say.
Regards,
Tim
On 15/Jun/2010 12:05, tellison@apache.org wrote:
> Author: tellison
> Date: Tue Jun 15 11:05:28 2010
> New Revision: 954817
>
> URL: http://svn.apache.org/viewvc?rev=954817&view=rev
> Log:
> Apply patch for HARMONY-6470 ([classlib][luni] Scanner will exhaust all heap memory when parse a large file)
>
> Includes a (relatively) long running test case.
>
> Added:
> harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java (with props)
> Modified:
> harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
>