You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2011/03/23 12:57:19 UTC
svn commit: r1084552 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/benchmark/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/
lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/
solr/
Author: doronc
Date: Wed Mar 23 11:57:18 2011
New Revision: 1084552
URL: http://svn.apache.org/viewvc?rev=1084552&view=rev
Log:
LUCENE-2980: Benchmark's ContentSource made insensitive to letter case of file suffix - merge from trunk to 3x.
Added:
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/ContentSourceTest.java
- copied unchanged from r1084544, lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/ContentSourceTest.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt?rev=1084552&r1=1084551&r2=1084552&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt Wed Mar 23 11:57:18 2011
@@ -2,6 +2,12 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
+03/23/2011
+ LUCENE-2980: Benchmark's ContentSource no more requires lower case file suffixes
+ for detecting file type (gzip/bzip2/text). As part of this fix worked around an
+ issue with gzip input streams which were remaining open (See COMPRESS-127).
+ (Doron Cohen)
+
03/22/2011
LUCENE-2978: Upgrade benchmark's commons-compress from 1.0 to 1.1 as
the move of gzip decompression in LUCENE-1540 from Java's GZipInputStream
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java?rev=1084552&r1=1084551&r2=1084552&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java Wed Mar 23 11:57:18 2011
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
import org.apache.commons.compress.compressors.CompressorException;
@@ -128,23 +129,49 @@ public abstract class ContentSource {
int idx = fileName.lastIndexOf('.');
String type = null;
if (idx != -1) {
- type = extensionToType.get(fileName.substring(idx));
+ type = extensionToType.get(fileName.substring(idx).toLowerCase(Locale.ENGLISH));
}
- try {
- if (type!=null) { // bzip or gzip
- return csFactory.createCompressorInputStream(type, is);
- }
- } catch (CompressorException e) {
- IOException ioe = new IOException(e.getMessage());
- ioe.initCause(e);
- throw ioe;
- }
+ if (type!=null) { // bzip or gzip
+ try {
+ return closableCompressorInputStream(type,is);
+ } catch (CompressorException e) {
+ IOException ioe = new IOException(e.getMessage());
+ ioe.initCause(e);
+ throw ioe;
+ }
+ }
return is;
}
/**
+ * Wrap the compressor input stream so that calling close will also close
+ * the underlying stream - workaround for CommonsCompress bug (COMPRESS-127).
+ */
+ private InputStream closableCompressorInputStream(String type, final InputStream is) throws CompressorException {
+ final InputStream delegee = csFactory.createCompressorInputStream(type, is);
+ if (!type.equals(CompressorStreamFactory.GZIP)) {
+ return delegee; //compressor bug affects only gzip
+ }
+ return new InputStream() {
+ @Override public int read() throws IOException { return delegee.read(); }
+ @Override public int read(byte[] b) throws IOException { return delegee.read(b); }
+ @Override public int available() throws IOException { return delegee.available(); }
+ @Override public synchronized void mark(int readlimit) { delegee.mark(readlimit); }
+ @Override public boolean markSupported() { return delegee.markSupported(); }
+ @Override public int read(byte[] b, int off, int len) throws IOException { return delegee.read(b, off, len); }
+ @Override public synchronized void reset() throws IOException { delegee.reset(); }
+ @Override public long skip(long n) throws IOException { return delegee.skip(n); }
+ @Override
+ public void close() throws IOException {
+ delegee.close();
+ is.close();
+ }
+ };
+ }
+
+ /**
* Returns true whether it's time to log a message (depending on verbose and
* the number of documents generated).
*/