You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2022/04/28 08:36:22 UTC
[maven-indexer] branch MINDEXER-121-chunkreader-leak created (now 5f57797)
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a change to branch MINDEXER-121-chunkreader-leak
in repository https://gitbox.apache.org/repos/asf/maven-indexer.git
at 5f57797 [MINDEXER-121] Rework resource handling in IndexReader
This branch includes the following new commits:
new 5f57797 [MINDEXER-121] Rework resource handling in IndexReader
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[maven-indexer] 01/01: [MINDEXER-121] Rework resource handling in IndexReader
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch MINDEXER-121-chunkreader-leak
in repository https://gitbox.apache.org/repos/asf/maven-indexer.git
commit 5f5779795e8e2aec05dc584170ea8311258fef83
Author: Tamas Cservenak <ta...@cservenak.net>
AuthorDate: Thu Apr 28 10:35:24 2022 +0200
[MINDEXER-121] Rework resource handling in IndexReader
Reworked completely resource handling in reader.
Updated all the tests with new explicity usages, also
removed all the deprecated code as well.
---
.../org/apache/maven/index/reader/ChunkReader.java | 16 +-
.../org/apache/maven/index/reader/ChunkWriter.java | 4 +-
.../org/apache/maven/index/reader/IndexReader.java | 66 +++--
.../org/apache/maven/index/reader/IndexWriter.java | 45 +--
.../java/org/apache/maven/index/reader/Record.java | 2 +-
.../apache/maven/index/reader/RecordCompactor.java | 2 +-
.../apache/maven/index/reader/RecordExpander.java | 2 +-
.../apache/maven/index/reader/ResourceHandler.java | 2 +-
.../java/org/apache/maven/index/reader/Utils.java | 60 +---
.../index/reader/WritableResourceHandler.java | 2 +-
.../maven/index/reader/CachingResourceHandler.java | 1 -
.../apache/maven/index/reader/ChunkReaderTest.java | 73 +++--
.../index/reader/DirectoryResourceHandler.java | 88 +++---
.../maven/index/reader/HttpResourceHandler.java | 60 ++--
.../apache/maven/index/reader/IndexReaderTest.java | 318 +++++++++++----------
.../apache/maven/index/reader/IndexWriterTest.java | 102 +++----
.../org/apache/maven/index/reader/TestSupport.java | 60 +---
.../org/apache/maven/index/reader/TestUtils.java | 106 +++----
.../apache/maven/index/reader/TransformTest.java | 117 ++++----
19 files changed, 553 insertions(+), 573 deletions(-)
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
index c42c4ea..c7789be 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
@@ -34,7 +34,21 @@ import java.util.NoSuchElementException;
import java.util.zip.GZIPInputStream;
/**
- * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
+ * Maven Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
+ * Instances of this class MUST BE handled as resources (have them closed once done with them), it is user
+ * responsibility to close them, ideally in try-with-resource block.
+ *
+ * Example:
+ * <pre>
+ * Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ * while ( chunkReaders.hasNext() )
+ * {
+ * try ( ChunkReader chunkReader = chunkReaders.next() )
+ * {
+ * ... use chunk reader
+ * }
+ * }
+ * </pre>
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
index c3c8e18..17c4a52 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
@@ -30,7 +30,9 @@ import java.util.Map;
import java.util.zip.GZIPOutputStream;
/**
- * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ * Maven Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ * Instances of this class MUST BE handled as resources (have them closed once done with them), it is user
+ * responsibility to close them, ideally in try-with-resource block.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
index d546f8a..795242e 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
@@ -19,8 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-
import java.io.Closeable;
import java.io.IOException;
import java.text.ParseException;
@@ -31,19 +29,24 @@ import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.maven.index.reader.ResourceHandler.Resource;
import static org.apache.maven.index.reader.Utils.loadProperties;
import static org.apache.maven.index.reader.Utils.storeProperties;
/**
- * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
+ * Maven Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
* read all the required records.
*
* @since 5.1.2
*/
public class IndexReader
- implements Iterable<ChunkReader>, Closeable
+ implements Iterable<ChunkReader>, Closeable
{
+ private final AtomicBoolean closed;
+
private final WritableResourceHandler local;
private final ResourceHandler remote;
@@ -61,9 +64,10 @@ public class IndexReader
private final List<String> chunkNames;
public IndexReader( final WritableResourceHandler local, final ResourceHandler remote )
- throws IOException
+ throws IOException
{
Objects.requireNonNull( remote, "remote resource handler null" );
+ this.closed = new AtomicBoolean( false );
this.local = local;
this.remote = remote;
remoteIndexProperties = loadProperties( remote.locate( Utils.INDEX_FILE_PREFIX + ".properties" ) );
@@ -85,8 +89,8 @@ public class IndexReader
if ( remoteIndexId == null || !remoteIndexId.equals( localIndexId ) )
{
throw new IllegalArgumentException(
- "local and remote index IDs does not match or is null: " + localIndexId + ", "
- + remoteIndexId );
+ "local and remote index IDs does not match or is null: " + localIndexId + ", "
+ + remoteIndexId );
}
this.indexId = localIndexId;
this.incremental = canRetrieveAllChunks();
@@ -105,7 +109,7 @@ public class IndexReader
this.incremental = false;
}
this.publishedTimestamp =
- Utils.INDEX_DATE_FORMAT.parse( remoteIndexProperties.getProperty( "nexus.index.timestamp" ) );
+ Utils.INDEX_DATE_FORMAT.parse( remoteIndexProperties.getProperty( "nexus.index.timestamp" ) );
this.chunkNames = calculateChunkNames();
}
catch ( ParseException e )
@@ -155,29 +159,32 @@ public class IndexReader
* consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents
* to prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this
* method should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of
- * this class should be closed manually.
+ * this class should be closed manually. This method acts only of first call, all the possible subsequent calls
+ * end up doing nothing.
*/
public void close()
- throws IOException
+ throws IOException
{
- remote.close();
- if ( local != null )
+ if ( closed.compareAndSet( false, true ) )
{
- try
- {
- syncLocalWithRemote();
- }
- finally
+ remote.close();
+ if ( local != null )
{
- local.close();
+ try
+ {
+ syncLocalWithRemote();
+ }
+ finally
+ {
+ local.close();
+ }
}
}
}
/**
* Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental)
- * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current
- * {@link ChunkReader} if aborting.
+ * updates from the index. It is caller responsibility to close each returned {@link ChunkReader}!
*/
public Iterator<ChunkReader> iterator()
{
@@ -189,7 +196,7 @@ public class IndexReader
* for future incremental updates.
*/
private void syncLocalWithRemote()
- throws IOException
+ throws IOException
{
storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), remoteIndexProperties );
}
@@ -234,7 +241,7 @@ public class IndexReader
try
{
int localLastIncremental =
- Integer.parseInt( localIndexProperties.getProperty( "nexus.index.last-incremental" ) );
+ Integer.parseInt( localIndexProperties.getProperty( "nexus.index.last-incremental" ) );
String currentLocalCounter = String.valueOf( localLastIncremental );
String nextLocalCounter = String.valueOf( localLastIncremental + 1 );
// check remote props for existence of current or next chunk after local
@@ -263,16 +270,12 @@ public class IndexReader
* is being consumed.
*/
private static class ChunkReaderIterator
- implements Iterator<ChunkReader>
+ implements Iterator<ChunkReader>
{
private final ResourceHandler resourceHandler;
private final Iterator<String> chunkNamesIterator;
- private Resource currentResource;
-
- private ChunkReader currentChunkReader;
-
private ChunkReaderIterator( final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator )
{
this.resourceHandler = resourceHandler;
@@ -289,13 +292,8 @@ public class IndexReader
String chunkName = chunkNamesIterator.next();
try
{
- if ( currentChunkReader != null )
- {
- currentChunkReader.close();
- }
- currentResource = resourceHandler.locate( chunkName );
- currentChunkReader = new ChunkReader( chunkName, currentResource.read() );
- return currentChunkReader;
+ Resource currentResource = resourceHandler.locate( chunkName );
+ return new ChunkReader( chunkName, currentResource.read() );
}
catch ( IOException e )
{
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
index c470411..5b33f52 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
@@ -19,8 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
-
import java.io.Closeable;
import java.io.IOException;
import java.text.ParseException;
@@ -30,12 +28,15 @@ import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
import static org.apache.maven.index.reader.Utils.loadProperties;
import static org.apache.maven.index.reader.Utils.storeProperties;
/**
- * Maven 2 Index writer that writes chunk and maintains published property file.
+ * Maven Index writer that writes chunk and maintains published property file.
* <p/>
* <strong>Currently no incremental update is supported, as the deleteion states should be maintained by
* caller</strong>. Hence, this writer will always produce the "main" chunk only.
@@ -43,10 +44,12 @@ import static org.apache.maven.index.reader.Utils.storeProperties;
* @since 5.1.2
*/
public class IndexWriter
- implements Closeable
+ implements Closeable
{
private static final int INDEX_V1 = 1;
+ private final AtomicBoolean closed;
+
private final WritableResourceHandler local;
private final Properties localIndexProperties;
@@ -58,10 +61,11 @@ public class IndexWriter
private final String nextChunkName;
public IndexWriter( final WritableResourceHandler local, final String indexId, final boolean incrementalSupported )
- throws IOException
+ throws IOException
{
Objects.requireNonNull( local, "local resource handler null" );
Objects.requireNonNull( indexId, "indexId null" );
+ this.closed = new AtomicBoolean( false );
this.local = local;
Properties indexProperties = loadProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ) );
if ( incrementalSupported && indexProperties != null )
@@ -72,7 +76,7 @@ public class IndexWriter
if ( localIndexId == null || !localIndexId.equals( indexId ) )
{
throw new IllegalArgumentException(
- "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + indexId );
+ "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + indexId );
}
this.incremental = true;
this.nextChunkCounter = calculateNextChunkCounter();
@@ -149,7 +153,7 @@ public class IndexWriter
* Writes out the record iterator and returns the written record count.
*/
public int writeChunk( final Iterator<Map<String, String>> iterator )
- throws IOException
+ throws IOException
{
int written;
@@ -172,23 +176,28 @@ public class IndexWriter
* Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients
* becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this
* method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of
- * this class should be closed manually.
+ * this class should be closed manually. This method acts only of first call, all the possible subsequent calls
+ * end up doing nothing.
*/
public void close()
- throws IOException
+ throws IOException
{
- try
+ if ( closed.compareAndSet( false, true ) )
{
- if ( incremental )
+ try
{
- localIndexProperties.setProperty( "nexus.index.last-incremental", nextChunkCounter );
+ if ( incremental )
+ {
+ localIndexProperties.setProperty( "nexus.index.last-incremental", nextChunkCounter );
+ }
+ localIndexProperties.setProperty( "nexus.index.timestamp",
+ Utils.INDEX_DATE_FORMAT.format( new Date() ) );
+ storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), localIndexProperties );
+ }
+ finally
+ {
+ local.close();
}
- localIndexProperties.setProperty( "nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format( new Date() ) );
- storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), localIndexProperties );
- }
- finally
- {
- local.close();
}
}
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
index c87bc0c..ea09c3b 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.Objects;
/**
- * Maven 2 Index record.
+ * Maven Index record.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
index 3e0e042..407caf4 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
@@ -30,7 +30,7 @@ import static org.apache.maven.index.reader.Utils.UINFO;
import static org.apache.maven.index.reader.Utils.nvl;
/**
- * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
+ * Maven Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
index a9d3c96..0a658c0 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
@@ -34,7 +34,7 @@ import static org.apache.maven.index.reader.Utils.renvl;
/**
- * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
+ * Maven Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
index 111e872..471f1c6 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.InputStream;
/**
- * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that
+ * Maven Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that
* implementations could perform any extra activity as FS locking or so (if uses FS as backing store). Is used by single
* thread only.
*
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
index e6a9afc..6149556 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
@@ -19,11 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.EntryKey;
-import org.apache.maven.index.reader.Record.Type;
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -35,6 +30,11 @@ import java.util.Properties;
import java.util.TimeZone;
import java.util.regex.Pattern;
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.apache.maven.index.reader.ResourceHandler.Resource;
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
+
/**
* Reusable code snippets and constants.
*
@@ -67,69 +67,35 @@ public final class Utils
public static final Pattern FS_PATTERN = Pattern.compile( Pattern.quote( FIELD_SEPARATOR ) );
- /**
- * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream.
- */
- public static Properties loadProperties( final InputStream inputStream )
- throws IOException
- {
- try
- {
- final Properties properties = new Properties();
- properties.load( inputStream );
- return properties;
- }
- finally
- {
- inputStream.close();
- }
- }
-
/**
* Creates and loads {@link Properties} from provided {@link Resource} if exists, and closes the resource. If not
* exists, returns {@code null}.
*/
public static Properties loadProperties( final Resource resource )
- throws IOException
+ throws IOException
{
final InputStream inputStream = resource.read();
if ( inputStream == null )
{
return null;
}
- return loadProperties( inputStream );
- }
-
- /**
- * Saves {@link Properties} to provided {@link OutputStream} and closes the stream.
- */
- public static void storeProperties( final OutputStream outputStream, final Properties properties )
- throws IOException
- {
- try
+ try ( InputStream is = inputStream )
{
- properties.store( outputStream, "Maven Indexer Writer" );
- }
- finally
- {
- outputStream.close();
+ final Properties properties = new Properties();
+ properties.load( is );
+ return properties;
}
}
-
/**
* Saves {@link Properties} to provided {@link WritableResource} and closes the resource.
*/
public static void storeProperties( final WritableResource writableResource, final Properties properties )
- throws IOException
+ throws IOException
{
- try
- {
- storeProperties( writableResource.write(), properties );
- }
- finally
+ try ( OutputStream outputStream = writableResource.write() )
{
- writableResource.close();
+ properties.store( outputStream, "Maven Indexer Writer" );
}
}
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
index 207b076..3580c47 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.OutputStream;
/**
- * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
+ * Maven Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
* updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link
* IndexReader} invocation. Is used by single thread only.
*
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
index 5e2dd32..dace48e 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
@@ -53,7 +53,6 @@ public class CachingResourceHandler
}
public Resource locate( final String name )
- throws IOException
{
if ( notFoundResources.contains( name ) )
{
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
index eb6a393..76ca7fa 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
@@ -19,10 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-import org.junit.Test;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -31,59 +27,62 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link ChunkReader}
*/
public class ChunkReaderTest
- extends TestSupport
+ extends TestSupport
{
@Test
public void simple()
- throws IOException
+ throws IOException
{
- final ChunkReader chunkReader = new ChunkReader( "full", testResourceHandler( "simple" ) //
- .locate( "nexus-maven-repository-index.gz" ).read() );
- final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
- assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ ChunkReader chunkReader = new ChunkReader( "full",
+ resourceHandler.locate( "nexus-maven-repository-index.gz" ).read() ) )
+ {
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ }
}
@Test
public void roundtrip()
- throws IOException
+ throws IOException
{
final Date published;
File tempChunkFile = createTempFile( "nexus-maven-repository-index.gz" );
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ ChunkReader chunkReader = new ChunkReader( "full",
+ resourceHandler.locate( "nexus-maven-repository-index.gz" ).read() );
+ ChunkWriter chunkWriter = new ChunkWriter( chunkReader.getName(), new FileOutputStream( tempChunkFile ),
+ 1, new Date() ) )
{
- final Resource resource = testResourceHandler( "simple" ) //
- .locate( "nexus-maven-repository-index.gz" );
-
- try (ChunkReader chunkReader = new ChunkReader( "full",
- resource.read() ); ChunkWriter chunkWriter = new ChunkWriter(
- chunkReader.getName(), //
- new FileOutputStream( tempChunkFile ), 1, new Date() ))
- {
- chunkWriter.writeChunk( chunkReader.iterator() );
- published = chunkWriter.getTimestamp();
- }
-
+ chunkWriter.writeChunk( chunkReader.iterator() );
+ published = chunkWriter.getTimestamp();
}
- final ChunkReader chunkReader = new ChunkReader( "full", new FileInputStream( tempChunkFile ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( published.getTime() ) );
- final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
- assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ try ( ChunkReader chunkReader = new ChunkReader( "full", new FileInputStream( tempChunkFile ) ) )
+ {
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( published.getTime() ) );
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ }
}
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
index eaa49e8..2b205df 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
@@ -34,56 +34,64 @@ import java.io.OutputStream;
* by name from specified existing directory.
*/
public class DirectoryResourceHandler
- implements WritableResourceHandler
+ implements WritableResourceHandler
{
- private final File rootDirectory;
+ private final File rootDirectory;
- public DirectoryResourceHandler(final File rootDirectory) {
- if (rootDirectory == null) {
- throw new NullPointerException("null rootDirectory");
+ public DirectoryResourceHandler( final File rootDirectory )
+ {
+ if ( rootDirectory == null )
+ {
+ throw new NullPointerException( "null rootDirectory" );
+ }
+ if ( !rootDirectory.isDirectory() )
+ {
+ throw new IllegalArgumentException( "rootDirectory exists and is not a directory" );
+ }
+ this.rootDirectory = rootDirectory;
}
- if (!rootDirectory.isDirectory()) {
- throw new IllegalArgumentException("rootDirectory exists and is not a directory");
- }
- this.rootDirectory = rootDirectory;
- }
- public File getRootDirectory() {
- return rootDirectory;
- }
+ public WritableResource locate( final String name )
+ {
+ return new FileResource( new File( rootDirectory, name ) );
+ }
- public WritableResource locate(final String name) throws IOException {
- return new FileResource( new File( rootDirectory, name ) );
- }
+ public void close() throws IOException
+ {
+ // nop
+ }
- public void close() throws IOException {
- // nop
- }
+ private static class FileResource
+ implements WritableResource
+ {
+ private final File file;
- private static class FileResource
- implements WritableResource
- {
- private final File file;
+ private FileResource( final File file )
+ {
+ this.file = file;
+ }
- private FileResource(final File file) {
- this.file = file;
- }
+ public InputStream read() throws IOException
+ {
+ try
+ {
+ return new BufferedInputStream( new FileInputStream( file ) );
+ }
+ catch ( FileNotFoundException e )
+ {
+ return null;
+ }
+ }
- public InputStream read() throws IOException {
- try {
- return new BufferedInputStream(new FileInputStream(file));
- } catch (FileNotFoundException e) {
- return null;
- }
- }
-
- public OutputStream write() throws IOException {
- return new BufferedOutputStream(new FileOutputStream(file));
- }
+ public OutputStream write() throws IOException
+ {
+ return new BufferedOutputStream( new FileOutputStream( file ) );
+ }
- public void close() throws IOException {
- // nop
+ public void close() throws IOException
+ {
+ // nop
+ }
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
index e19d949..fe1043b 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
@@ -32,40 +32,46 @@ import java.net.URL;
* handle any advanced cases, like redirects, authentication, etc.
*/
public class HttpResourceHandler
- implements ResourceHandler
+ implements ResourceHandler
{
- private final URI root;
+ private final URI root;
- public HttpResourceHandler(final URL root) throws URISyntaxException {
- if (root == null) {
- throw new NullPointerException("root URL null");
+ public HttpResourceHandler( final URL root ) throws URISyntaxException
+ {
+ if ( root == null )
+ {
+ throw new NullPointerException( "root URL null" );
+ }
+ this.root = root.toURI();
}
- this.root = root.toURI();
- }
- public Resource locate(final String name) throws IOException {
- return new HttpResource(name);
- }
+ public Resource locate( final String name )
+ {
+ return new HttpResource( name );
+ }
- private class HttpResource
- implements Resource
- {
- private final String name;
+ private class HttpResource
+ implements Resource
+ {
+ private final String name;
- private HttpResource(final String name) {
- this.name = name;
- }
+ private HttpResource( final String name )
+ {
+ this.name = name;
+ }
- public InputStream read() throws IOException {
- URL target = root.resolve(name).toURL();
- HttpURLConnection conn = (HttpURLConnection) target.openConnection();
- conn.setRequestMethod("GET");
- conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0");
- return new BufferedInputStream(conn.getInputStream());
+ public InputStream read() throws IOException
+ {
+ URL target = root.resolve( name ).toURL();
+ HttpURLConnection conn = (HttpURLConnection) target.openConnection();
+ conn.setRequestMethod( "GET" );
+ conn.setRequestProperty( "User-Agent", "ASF Maven-Indexer-Reader/1.0" );
+ return new BufferedInputStream( conn.getInputStream() );
+ }
}
- }
- public void close() throws IOException {
- // nop
- }
+ public void close() throws IOException
+ {
+ // nop
+ }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
index cc5c2d2..0fb7867 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
@@ -19,187 +19,215 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.Ignore;
-import org.junit.Test;
-
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
+import java.util.Iterator;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import static com.google.common.collect.Iterables.transform;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Ignore;
+import org.junit.Test;
+
import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.hamcrest.core.IsNot.not;
import static org.hamcrest.core.IsNull.nullValue;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link IndexReader}
*/
public class IndexReaderTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void simple() throws IOException {
- try ( IndexReader indexReader = new IndexReader( null, testResourceHandler( "simple" ) ) )
+ @Test
+ public void simple() throws IOException
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
- for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ try ( IndexReader indexReader = new IndexReader( null, testResourceHandler( "simple" ) ) )
{
- records++;
- }
- }
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ int records = 0;
+
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
+ for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
+ .map( expandFunction )
+ .collect( Collectors.toList() ) )
+ {
+ records++;
+ }
+ }
+ }
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records, equalTo( 5 ) );
+ }
}
- }
- @Test
- public void simpleWithLocal() throws IOException {
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- try ( IndexReader indexReader = new IndexReader( writableResourceHandler, testResourceHandler( "simple" ) ) )
+ @Test
+ public void simpleWithLocal() throws IOException
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
- for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
{
- records++;
- }
- }
+ try ( IndexReader indexReader = new IndexReader( writableResourceHandler,
+ testResourceHandler( "simple" ) ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(),
+ equalTo( Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ int records = 0;
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
- }
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
+ for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
+ .map( expandFunction )
+ .collect( Collectors.toList() ) )
+ {
+ records++;
+ }
+ }
+ }
- assertThat(writableResourceHandler.locate("nexus-maven-repository-index.properties").read(), not(nullValue()));
- }
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records, equalTo( 5 ) );
+ }
+ assertThat( writableResourceHandler.locate( "nexus-maven-repository-index.properties" ).read(),
+ not( nullValue() ) );
+ }
+ }
- @Test
- public void roundtrip() throws IOException {
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- Date published;
+ @Test
+ public void roundtrip() throws IOException
{
- final IndexReader indexReader = new IndexReader(
- null,
- testResourceHandler("simple")
- );
- final IndexWriter indexWriter = new IndexWriter(
- writableResourceHandler,
- indexReader.getIndexId(),
- false
- );
- try {
- for (ChunkReader chunkReader : indexReader) {
- indexWriter.writeChunk(chunkReader.iterator());
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ Date published;
+ {
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ IndexReader indexReader = new IndexReader( null, resourceHandler );
+ IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexReader.getIndexId(),
+ false ) )
+ {
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ indexWriter.writeChunk( chunkReader.iterator() );
+ }
+ }
+ indexWriter.close(); // must close to store properties, so next call returns non-null
+ published = indexWriter.getPublishedTimestamp();
+ }
+ }
+
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( published.getTime() ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ AtomicInteger records = new AtomicInteger( 0 );
+
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ chunkReader.forEach( r -> records.incrementAndGet() );
+ }
+ }
+
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records.get(), equalTo( 5 ) );
+ }
}
- }
- finally {
- indexWriter.close();
- published = indexWriter.getPublishedTimestamp();
- indexReader.close();
- }
}
- try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ /**
+ * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your
+ * IP address being banned. You were warned!
+ */
+ @Test
+ @Ignore( "For eyes only" )
+ public void central() throws Exception
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( published.getTime() ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for ( Record record : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ // local index location, against which we perform incremental updates
+ final File indexDir = createTempDirectory();
+ // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early
+ final File cacheDir = createTempDirectory();
+
+ final PrintWriter writer = new PrintWriter( System.out, true );
+ final WritableResourceHandler local = new DirectoryResourceHandler( indexDir );
+ final CachingResourceHandler remote = new CachingResourceHandler(
+ new DirectoryResourceHandler( cacheDir ),
+ new HttpResourceHandler( new URL( "http://repo1.maven.org/maven2/.index/" ) )
+ );
+ final IndexReader indexReader = new IndexReader( local, remote );
+ try
{
- records++;
- }
- }
+ writer.println( "indexRepoId=" + indexReader.getIndexId() );
+ writer.println( "indexLastPublished=" + indexReader.getPublishedTimestamp() );
+ writer.println( "isIncremental=" + indexReader.isIncremental() );
+ writer.println( "indexRequiredChunkNames=" + indexReader.getChunkNames() );
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
- }
- }
-
- /**
- * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your
- * IP address being banned. You were warned!
- */
- @Test
- @Ignore("For eyes only")
- public void central() throws Exception {
- // local index location, against which we perform incremental updates
- final File indexDir = createTempDirectory();
- // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early
- final File cacheDir = createTempDirectory();
-
- final PrintWriter writer = new PrintWriter(System.out, true);
- final WritableResourceHandler local = new DirectoryResourceHandler(indexDir);
- final CachingResourceHandler remote = new CachingResourceHandler(
- new DirectoryResourceHandler(cacheDir),
- new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/"))
- );
- final IndexReader indexReader = new IndexReader(local, remote);
- try {
- writer.println("indexRepoId=" + indexReader.getIndexId());
- writer.println("indexLastPublished=" + indexReader.getPublishedTimestamp());
- writer.println("isIncremental=" + indexReader.isIncremental());
- writer.println("indexRequiredChunkNames=" + indexReader.getChunkNames());
- for (ChunkReader chunkReader : indexReader) {
- writer.println("chunkName=" + chunkReader.getName());
- writer.println("chunkVersion=" + chunkReader.getVersion());
- writer.println("chunkPublished=" + chunkReader.getTimestamp());
- writer.println("Chunk stats:");
- Map<Type, Integer> stats = countRecordsByType(chunkReader);
- for (Map.Entry<Type, Integer> entry : stats.entrySet()) {
- writer.println(entry.getKey() + " = " + entry.getValue());
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ writer.println( "chunkName=" + chunkReader.getName() );
+ writer.println( "chunkVersion=" + chunkReader.getVersion() );
+ writer.println( "chunkPublished=" + chunkReader.getTimestamp() );
+ writer.println( "Chunk stats:" );
+ Map<Type, Integer> stats = countRecordsByType( chunkReader );
+ for ( Map.Entry<Type, Integer> entry : stats.entrySet() )
+ {
+ writer.println( entry.getKey() + " = " + entry.getValue() );
+ }
+ writer.println( "= = = = = =" );
+ }
+ }
+ }
+ finally
+ {
+ indexReader.close();
+ remote.close();
+ local.close();
}
- writer.println("= = = = = =");
- }
- }
- finally {
- indexReader.close();
- remote.close();
- local.close();
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
index 48aadb2..fd97cbc 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
@@ -19,79 +19,57 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.junit.Test;
-
import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
-import static com.google.common.collect.Iterables.transform;
-import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link IndexWriter}
*/
public class IndexWriterTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void roundtrip() throws IOException {
- IndexReader indexReader;
- IndexWriter indexWriter;
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ @Test
+ public void roundtrip() throws IOException
+ {
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ try ( IndexReader indexReader = new IndexReader( null, resourceHandler );
+ IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexReader.getIndexId(),
+ false ) )
+ {
+ for ( ChunkReader chunkReader : indexReader )
+ {
+ indexWriter.writeChunk( chunkReader.iterator() );
+ }
+ }
- // write it once
- indexReader = new IndexReader(
- null,
- testResourceHandler("simple")
- );
- indexWriter = new IndexWriter(
- writableResourceHandler,
- indexReader.getIndexId(),
- false
- );
- try {
- for (ChunkReader chunkReader : indexReader) {
- indexWriter.writeChunk(chunkReader.iterator());
- }
- }
- finally {
- indexWriter.close();
- indexReader.close();
- }
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(),
+ equalTo( Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ AtomicInteger records = new AtomicInteger( 0 );
+ for ( ChunkReader chunkReader : indexReader )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ chunkReader.forEach( r -> records.incrementAndGet() );
+ }
- // read what we wrote out
- indexReader = new IndexReader(
- null,
- writableResourceHandler
- );
- try {
- assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
- // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
- assertThat(indexReader.isIncremental(), equalTo(false));
- assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
- int chunks = 0;
- int records = 0;
- for (ChunkReader chunkReader : indexReader) {
- chunks++;
- assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
- assertThat(chunkReader.getVersion(), equalTo(1));
- // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for (Record record : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) ) {
- records++;
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records.get(), equalTo( 5 ) );
+ }
}
- }
-
- assertThat(chunks, equalTo(1));
- assertThat(records, equalTo(5));
- }
- finally {
- indexReader.close();
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
index 72f1040..0f299e5 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
@@ -19,23 +19,22 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* Test support.
@@ -54,7 +53,7 @@ public class TestSupport
*/
@Before
public void setup()
- throws IOException
+ throws IOException
{
this.tempDir = Files.createTempDirectory( getClass().getSimpleName() + ".temp" ).toFile();
this.directoryResourceHandlers = new ArrayList<>();
@@ -65,7 +64,7 @@ public class TestSupport
*/
@After
public void cleanup()
- throws IOException
+ throws IOException
{
for ( DirectoryResourceHandler directoryResourceHandler : directoryResourceHandlers )
{
@@ -78,7 +77,6 @@ public class TestSupport
* Creates a temp file within {@link #tempDir} with given name.
*/
protected File createTempFile( final String name )
- throws IOException
{
File file = new File( tempDir, name );
file.deleteOnExit();
@@ -89,7 +87,7 @@ public class TestSupport
* Creates a temp directory within {@link #tempDir}.
*/
protected File createTempDirectory()
- throws IOException
+ throws IOException
{
return Files.createTempDirectory( tempDir.toPath(), testName.getMethodName() + "-dir" ).toFile();
}
@@ -98,7 +96,7 @@ public class TestSupport
* Creates an empty {@link DirectoryResourceHandler}.
*/
protected WritableResourceHandler createWritableResourceHandler()
- throws IOException
+ throws IOException
{
DirectoryResourceHandler result = new DirectoryResourceHandler( createTempDirectory() );
directoryResourceHandlers.add( result );
@@ -110,7 +108,6 @@ public class TestSupport
* name.
*/
protected ResourceHandler testResourceHandler( final String name )
- throws IOException
{
DirectoryResourceHandler result = new DirectoryResourceHandler( new File( "src/test/resources/" + name ) );
directoryResourceHandlers.add( result );
@@ -121,7 +118,7 @@ public class TestSupport
* Consumes {@link ChunkReader} and creates a map "by type" with records.
*/
protected Map<Type, List<Record>> loadRecordsByType( final ChunkReader chunkReader )
- throws IOException
+ throws IOException
{
HashMap<Type, List<Record>> stat = new HashMap<>();
try
@@ -150,7 +147,7 @@ public class TestSupport
* Consumes {@link ChunkReader} and creates a map "by type" with record type counts.
*/
protected Map<Type, Integer> countRecordsByType( final ChunkReader chunkReader )
- throws IOException
+ throws IOException
{
HashMap<Type, Integer> stat = new HashMap<>();
try
@@ -173,35 +170,4 @@ public class TestSupport
}
return stat;
}
-
- /**
- * Delete recursively.
- */
- private static boolean delete( final File file )
- {
- if ( file == null )
- {
- return false;
- }
- if ( !file.exists() )
- {
- return true;
- }
- if ( file.isDirectory() )
- {
- String[] list = file.list();
- if ( list != null )
- {
- for ( String s : list )
- {
- File entry = new File( file, s );
- if ( !delete( entry ) )
- {
- return false;
- }
- }
- }
- }
- return file.delete();
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
index 2e6912b..8ae5eef 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
@@ -19,76 +19,80 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-
import java.util.Map;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import org.apache.maven.index.reader.Record.Type;
+
import static com.google.common.collect.Iterables.concat;
import static java.util.Collections.singletonList;
-import static org.apache.maven.index.reader.Utils.*;
+import static org.apache.maven.index.reader.Utils.allGroups;
+import static org.apache.maven.index.reader.Utils.descriptor;
+import static org.apache.maven.index.reader.Utils.rootGroup;
+import static org.apache.maven.index.reader.Utils.rootGroups;
/**
* Helpers to transform records from one to another representation, and, some helpers for publishing using Guava.
*/
public final class TestUtils
{
- private TestUtils() {
- // nothing
- }
+ private TestUtils()
+ {
+ // nothing
+ }
- private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor();
+ private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor();
- private static final RecordExpander RECORD_EXPANDER = new RecordExpander();
+ private static final RecordExpander RECORD_EXPANDER = new RecordExpander();
- public static Function<Record, Map<String, String>> compactFunction = RECORD_COMPACTOR::apply;
+ public static Function<Record, Map<String, String>> compactFunction = RECORD_COMPACTOR::apply;
- public static Function<Map<String, String>, Record> expandFunction = RECORD_EXPANDER::apply;
+ public static Function<Map<String, String>, Record> expandFunction = RECORD_EXPANDER::apply;
- /**
- * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
- * all the caller is needed to provide {@link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
- * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
- * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
- * records.
- */
- public static Iterable<Record> decorate(final Iterable<Record> iterable,
- final String repoId)
- {
- final TreeSet<String> allGroupsSet = new TreeSet<>();
- final TreeSet<String> rootGroupsSet = new TreeSet<>();
- return StreamSupport.stream(
- concat( singletonList( descriptor( repoId ) ), iterable, singletonList( allGroups( allGroupsSet ) ),
- // placeholder, will be recreated at the end with proper content
- singletonList( rootGroups( rootGroupsSet ) )
- // placeholder, will be recreated at the end with proper content
- ).spliterator(), false ).map( rec ->
+ /**
+ * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
+ * all the caller is needed to provide {@link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
+ * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
+ * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
+ * records.
+ */
+ public static Iterable<Record> decorate( final Iterable<Record> iterable,
+ final String repoId )
{
- if ( Type.DESCRIPTOR == rec.getType() )
- {
- return rec;
- }
- else if ( Type.ALL_GROUPS == rec.getType() )
- {
- return allGroups( allGroupsSet );
- }
- else if ( Type.ROOT_GROUPS == rec.getType() )
- {
- return rootGroups( rootGroupsSet );
- }
- else
- {
- final String groupId = rec.get( Record.GROUP_ID );
- if ( groupId != null )
+ final TreeSet<String> allGroupsSet = new TreeSet<>();
+ final TreeSet<String> rootGroupsSet = new TreeSet<>();
+ return StreamSupport.stream(
+ concat( singletonList( descriptor( repoId ) ), iterable, singletonList( allGroups( allGroupsSet ) ),
+ // placeholder, will be recreated at the end with proper content
+ singletonList( rootGroups( rootGroupsSet ) )
+ // placeholder, will be recreated at the end with proper content
+ ).spliterator(), false ).map( rec ->
{
- allGroupsSet.add( groupId );
- rootGroupsSet.add( rootGroup( groupId ) );
- }
- return rec;
- }
- } ).collect( Collectors.toList() );
- }
+ if ( Type.DESCRIPTOR == rec.getType() )
+ {
+ return rec;
+ }
+ else if ( Type.ALL_GROUPS == rec.getType() )
+ {
+ return allGroups( allGroupsSet );
+ }
+ else if ( Type.ROOT_GROUPS == rec.getType() )
+ {
+ return rootGroups( rootGroupsSet );
+ }
+ else
+ {
+ final String groupId = rec.get( Record.GROUP_ID );
+ if ( groupId != null )
+ {
+ allGroupsSet.add( groupId );
+ rootGroupsSet.add( rootGroup( groupId ) );
+ }
+ return rec;
+ }
+ } ).collect( Collectors.toList() );
+ }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
index 1ea6155..4817b7f 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
@@ -19,83 +19,86 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.EntryKey;
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.Test;
-
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import static com.google.common.collect.Iterables.transform;
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
import static org.apache.maven.index.reader.TestUtils.compactFunction;
import static org.apache.maven.index.reader.TestUtils.decorate;
import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
- * UT for {@link RecordCompactor} and {@linl RecordExpander}.
+ * UT for {@link RecordCompactor} and {@link RecordExpander}.
*/
public class TransformTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void decorateAndTransform() throws IOException {
- final String indexId = "test";
- final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache"));
- final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo"));
- final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar"));
+ @Test
+ public void decorateAndTransform() throws IOException
+ {
+ final String indexId = "test";
+ final Record r1 = new Record( Type.ARTIFACT_ADD, artifactMap( "org.apache" ) );
+ final Record r2 = new Record( Type.ARTIFACT_ADD, artifactMap( "org.foo" ) );
+ final Record r3 = new Record( Type.ARTIFACT_ADD, artifactMap( "com.bar" ) );
- Iterable<Map<String, String>> iterable = StreamSupport.stream(
- decorate( Arrays.asList( r1, r2, r3 ), indexId ).spliterator(), false )
- .map( compactFunction )
- .collect( Collectors.toList() );
+ Iterable<Map<String, String>> iterable = StreamSupport.stream(
+ decorate( Arrays.asList( r1, r2, r3 ), indexId ).spliterator(), false )
+ .map( compactFunction )
+ .collect( Collectors.toList() );
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- try {
- IndexWriter indexWriter = new IndexWriter(
- writableResourceHandler,
- indexId,
- false
- );
- indexWriter.writeChunk(iterable.iterator());
- indexWriter.close();
- }
- finally {
- writableResourceHandler.close();
- }
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ try ( IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexId, false ) )
+ {
+ indexWriter.writeChunk( iterable.iterator() );
+ }
- IndexReader indexReader = new IndexReader(null, writableResourceHandler);
- assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
- ChunkReader chunkReader = indexReader.iterator().next();
- final Map<Type, List<Record>> recordTypes = loadRecordsByType(chunkReader);
- assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(3));
- assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ try ( ChunkReader chunkReader = indexReader.iterator().next() )
+ {
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 3 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
- assertThat(recordTypes.get(Type.ROOT_GROUPS).get(0).get(Record.ROOT_GROUPS), equalTo(new String[] {"com","org"}));
- assertThat(recordTypes.get(Type.ALL_GROUPS).get(0).get(Record.ALL_GROUPS), equalTo(new String[] {"com.bar", "org.apache", "org.foo"}));
- }
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).get( 0 ).get( Record.ROOT_GROUPS ),
+ equalTo( new String[] {"com", "org"} ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).get( 0 ).get( Record.ALL_GROUPS ),
+ equalTo( new String[] {"com.bar", "org.apache", "org.foo"} ) );
+ }
+ }
+ }
+ }
- private Map<EntryKey, Object> artifactMap(final String groupId) {
- final HashMap<EntryKey, Object> result = new HashMap<>();
- result.put(Record.GROUP_ID, groupId);
- result.put(Record.ARTIFACT_ID, "artifact");
- result.put(Record.VERSION, "1.0");
- result.put(Record.PACKAGING, "jar");
- result.put(Record.FILE_MODIFIED, System.currentTimeMillis());
- result.put(Record.FILE_SIZE, 123L);
- result.put(Record.FILE_EXTENSION, "jar");
- result.put(Record.HAS_SOURCES, Boolean.FALSE);
- result.put(Record.HAS_JAVADOC, Boolean.FALSE);
- result.put(Record.HAS_SIGNATURE, Boolean.FALSE);
- return result;
- }
+ private Map<EntryKey, Object> artifactMap( final String groupId )
+ {
+ final HashMap<EntryKey, Object> result = new HashMap<>();
+ result.put( Record.GROUP_ID, groupId );
+ result.put( Record.ARTIFACT_ID, "artifact" );
+ result.put( Record.VERSION, "1.0" );
+ result.put( Record.PACKAGING, "jar" );
+ result.put( Record.FILE_MODIFIED, System.currentTimeMillis() );
+ result.put( Record.FILE_SIZE, 123L );
+ result.put( Record.FILE_EXTENSION, "jar" );
+ result.put( Record.HAS_SOURCES, Boolean.FALSE );
+ result.put( Record.HAS_JAVADOC, Boolean.FALSE );
+ result.put( Record.HAS_SIGNATURE, Boolean.FALSE );
+ return result;
+ }
}