You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2022/04/28 08:36:23 UTC
[maven-indexer] 01/01: [MINDEXER-121] Rework resource handling in IndexReader
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch MINDEXER-121-chunkreader-leak
in repository https://gitbox.apache.org/repos/asf/maven-indexer.git
commit 5f5779795e8e2aec05dc584170ea8311258fef83
Author: Tamas Cservenak <ta...@cservenak.net>
AuthorDate: Thu Apr 28 10:35:24 2022 +0200
[MINDEXER-121] Rework resource handling in IndexReader
Reworked completely resource handling in reader.
Updated all the tests with new explicity usages, also
removed all the deprecated code as well.
---
.../org/apache/maven/index/reader/ChunkReader.java | 16 +-
.../org/apache/maven/index/reader/ChunkWriter.java | 4 +-
.../org/apache/maven/index/reader/IndexReader.java | 66 +++--
.../org/apache/maven/index/reader/IndexWriter.java | 45 +--
.../java/org/apache/maven/index/reader/Record.java | 2 +-
.../apache/maven/index/reader/RecordCompactor.java | 2 +-
.../apache/maven/index/reader/RecordExpander.java | 2 +-
.../apache/maven/index/reader/ResourceHandler.java | 2 +-
.../java/org/apache/maven/index/reader/Utils.java | 60 +---
.../index/reader/WritableResourceHandler.java | 2 +-
.../maven/index/reader/CachingResourceHandler.java | 1 -
.../apache/maven/index/reader/ChunkReaderTest.java | 73 +++--
.../index/reader/DirectoryResourceHandler.java | 88 +++---
.../maven/index/reader/HttpResourceHandler.java | 60 ++--
.../apache/maven/index/reader/IndexReaderTest.java | 318 +++++++++++----------
.../apache/maven/index/reader/IndexWriterTest.java | 102 +++----
.../org/apache/maven/index/reader/TestSupport.java | 60 +---
.../org/apache/maven/index/reader/TestUtils.java | 106 +++----
.../apache/maven/index/reader/TransformTest.java | 117 ++++----
19 files changed, 553 insertions(+), 573 deletions(-)
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
index c42c4ea..c7789be 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
@@ -34,7 +34,21 @@ import java.util.NoSuchElementException;
import java.util.zip.GZIPInputStream;
/**
- * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
+ * Maven Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
+ * Instances of this class MUST BE handled as resources (have them closed once done with them), it is user
+ * responsibility to close them, ideally in try-with-resource block.
+ *
+ * Example:
+ * <pre>
+ * Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ * while ( chunkReaders.hasNext() )
+ * {
+ * try ( ChunkReader chunkReader = chunkReaders.next() )
+ * {
+ * ... use chunk reader
+ * }
+ * }
+ * </pre>
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
index c3c8e18..17c4a52 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
@@ -30,7 +30,9 @@ import java.util.Map;
import java.util.zip.GZIPOutputStream;
/**
- * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ * Maven Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ * Instances of this class MUST BE handled as resources (have them closed once done with them), it is user
+ * responsibility to close them, ideally in try-with-resource block.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
index d546f8a..795242e 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
@@ -19,8 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-
import java.io.Closeable;
import java.io.IOException;
import java.text.ParseException;
@@ -31,19 +29,24 @@ import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.maven.index.reader.ResourceHandler.Resource;
import static org.apache.maven.index.reader.Utils.loadProperties;
import static org.apache.maven.index.reader.Utils.storeProperties;
/**
- * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
+ * Maven Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
* read all the required records.
*
* @since 5.1.2
*/
public class IndexReader
- implements Iterable<ChunkReader>, Closeable
+ implements Iterable<ChunkReader>, Closeable
{
+ private final AtomicBoolean closed;
+
private final WritableResourceHandler local;
private final ResourceHandler remote;
@@ -61,9 +64,10 @@ public class IndexReader
private final List<String> chunkNames;
public IndexReader( final WritableResourceHandler local, final ResourceHandler remote )
- throws IOException
+ throws IOException
{
Objects.requireNonNull( remote, "remote resource handler null" );
+ this.closed = new AtomicBoolean( false );
this.local = local;
this.remote = remote;
remoteIndexProperties = loadProperties( remote.locate( Utils.INDEX_FILE_PREFIX + ".properties" ) );
@@ -85,8 +89,8 @@ public class IndexReader
if ( remoteIndexId == null || !remoteIndexId.equals( localIndexId ) )
{
throw new IllegalArgumentException(
- "local and remote index IDs does not match or is null: " + localIndexId + ", "
- + remoteIndexId );
+ "local and remote index IDs does not match or is null: " + localIndexId + ", "
+ + remoteIndexId );
}
this.indexId = localIndexId;
this.incremental = canRetrieveAllChunks();
@@ -105,7 +109,7 @@ public class IndexReader
this.incremental = false;
}
this.publishedTimestamp =
- Utils.INDEX_DATE_FORMAT.parse( remoteIndexProperties.getProperty( "nexus.index.timestamp" ) );
+ Utils.INDEX_DATE_FORMAT.parse( remoteIndexProperties.getProperty( "nexus.index.timestamp" ) );
this.chunkNames = calculateChunkNames();
}
catch ( ParseException e )
@@ -155,29 +159,32 @@ public class IndexReader
* consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents
* to prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this
* method should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of
- * this class should be closed manually.
+ * this class should be closed manually. This method acts only of first call, all the possible subsequent calls
+ * end up doing nothing.
*/
public void close()
- throws IOException
+ throws IOException
{
- remote.close();
- if ( local != null )
+ if ( closed.compareAndSet( false, true ) )
{
- try
- {
- syncLocalWithRemote();
- }
- finally
+ remote.close();
+ if ( local != null )
{
- local.close();
+ try
+ {
+ syncLocalWithRemote();
+ }
+ finally
+ {
+ local.close();
+ }
}
}
}
/**
* Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental)
- * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current
- * {@link ChunkReader} if aborting.
+ * updates from the index. It is caller responsibility to close each returned {@link ChunkReader}!
*/
public Iterator<ChunkReader> iterator()
{
@@ -189,7 +196,7 @@ public class IndexReader
* for future incremental updates.
*/
private void syncLocalWithRemote()
- throws IOException
+ throws IOException
{
storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), remoteIndexProperties );
}
@@ -234,7 +241,7 @@ public class IndexReader
try
{
int localLastIncremental =
- Integer.parseInt( localIndexProperties.getProperty( "nexus.index.last-incremental" ) );
+ Integer.parseInt( localIndexProperties.getProperty( "nexus.index.last-incremental" ) );
String currentLocalCounter = String.valueOf( localLastIncremental );
String nextLocalCounter = String.valueOf( localLastIncremental + 1 );
// check remote props for existence of current or next chunk after local
@@ -263,16 +270,12 @@ public class IndexReader
* is being consumed.
*/
private static class ChunkReaderIterator
- implements Iterator<ChunkReader>
+ implements Iterator<ChunkReader>
{
private final ResourceHandler resourceHandler;
private final Iterator<String> chunkNamesIterator;
- private Resource currentResource;
-
- private ChunkReader currentChunkReader;
-
private ChunkReaderIterator( final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator )
{
this.resourceHandler = resourceHandler;
@@ -289,13 +292,8 @@ public class IndexReader
String chunkName = chunkNamesIterator.next();
try
{
- if ( currentChunkReader != null )
- {
- currentChunkReader.close();
- }
- currentResource = resourceHandler.locate( chunkName );
- currentChunkReader = new ChunkReader( chunkName, currentResource.read() );
- return currentChunkReader;
+ Resource currentResource = resourceHandler.locate( chunkName );
+ return new ChunkReader( chunkName, currentResource.read() );
}
catch ( IOException e )
{
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
index c470411..5b33f52 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
@@ -19,8 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
-
import java.io.Closeable;
import java.io.IOException;
import java.text.ParseException;
@@ -30,12 +28,15 @@ import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
import static org.apache.maven.index.reader.Utils.loadProperties;
import static org.apache.maven.index.reader.Utils.storeProperties;
/**
- * Maven 2 Index writer that writes chunk and maintains published property file.
+ * Maven Index writer that writes chunk and maintains published property file.
* <p/>
* <strong>Currently no incremental update is supported, as the deleteion states should be maintained by
* caller</strong>. Hence, this writer will always produce the "main" chunk only.
@@ -43,10 +44,12 @@ import static org.apache.maven.index.reader.Utils.storeProperties;
* @since 5.1.2
*/
public class IndexWriter
- implements Closeable
+ implements Closeable
{
private static final int INDEX_V1 = 1;
+ private final AtomicBoolean closed;
+
private final WritableResourceHandler local;
private final Properties localIndexProperties;
@@ -58,10 +61,11 @@ public class IndexWriter
private final String nextChunkName;
public IndexWriter( final WritableResourceHandler local, final String indexId, final boolean incrementalSupported )
- throws IOException
+ throws IOException
{
Objects.requireNonNull( local, "local resource handler null" );
Objects.requireNonNull( indexId, "indexId null" );
+ this.closed = new AtomicBoolean( false );
this.local = local;
Properties indexProperties = loadProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ) );
if ( incrementalSupported && indexProperties != null )
@@ -72,7 +76,7 @@ public class IndexWriter
if ( localIndexId == null || !localIndexId.equals( indexId ) )
{
throw new IllegalArgumentException(
- "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + indexId );
+ "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + indexId );
}
this.incremental = true;
this.nextChunkCounter = calculateNextChunkCounter();
@@ -149,7 +153,7 @@ public class IndexWriter
* Writes out the record iterator and returns the written record count.
*/
public int writeChunk( final Iterator<Map<String, String>> iterator )
- throws IOException
+ throws IOException
{
int written;
@@ -172,23 +176,28 @@ public class IndexWriter
* Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients
* becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this
* method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of
- * this class should be closed manually.
+ * this class should be closed manually. This method acts only of first call, all the possible subsequent calls
+ * end up doing nothing.
*/
public void close()
- throws IOException
+ throws IOException
{
- try
+ if ( closed.compareAndSet( false, true ) )
{
- if ( incremental )
+ try
{
- localIndexProperties.setProperty( "nexus.index.last-incremental", nextChunkCounter );
+ if ( incremental )
+ {
+ localIndexProperties.setProperty( "nexus.index.last-incremental", nextChunkCounter );
+ }
+ localIndexProperties.setProperty( "nexus.index.timestamp",
+ Utils.INDEX_DATE_FORMAT.format( new Date() ) );
+ storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), localIndexProperties );
+ }
+ finally
+ {
+ local.close();
}
- localIndexProperties.setProperty( "nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format( new Date() ) );
- storeProperties( local.locate( Utils.INDEX_FILE_PREFIX + ".properties" ), localIndexProperties );
- }
- finally
- {
- local.close();
}
}
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
index c87bc0c..ea09c3b 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.Objects;
/**
- * Maven 2 Index record.
+ * Maven Index record.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
index 3e0e042..407caf4 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
@@ -30,7 +30,7 @@ import static org.apache.maven.index.reader.Utils.UINFO;
import static org.apache.maven.index.reader.Utils.nvl;
/**
- * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
+ * Maven Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
index a9d3c96..0a658c0 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
@@ -34,7 +34,7 @@ import static org.apache.maven.index.reader.Utils.renvl;
/**
- * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
+ * Maven Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
*
* @since 5.1.2
*/
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
index 111e872..471f1c6 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.InputStream;
/**
- * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that
+ * Maven Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that
* implementations could perform any extra activity as FS locking or so (if uses FS as backing store). Is used by single
* thread only.
*
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
index e6a9afc..6149556 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
@@ -19,11 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.EntryKey;
-import org.apache.maven.index.reader.Record.Type;
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -35,6 +30,11 @@ import java.util.Properties;
import java.util.TimeZone;
import java.util.regex.Pattern;
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.apache.maven.index.reader.ResourceHandler.Resource;
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
+
/**
* Reusable code snippets and constants.
*
@@ -67,69 +67,35 @@ public final class Utils
public static final Pattern FS_PATTERN = Pattern.compile( Pattern.quote( FIELD_SEPARATOR ) );
- /**
- * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream.
- */
- public static Properties loadProperties( final InputStream inputStream )
- throws IOException
- {
- try
- {
- final Properties properties = new Properties();
- properties.load( inputStream );
- return properties;
- }
- finally
- {
- inputStream.close();
- }
- }
-
/**
* Creates and loads {@link Properties} from provided {@link Resource} if exists, and closes the resource. If not
* exists, returns {@code null}.
*/
public static Properties loadProperties( final Resource resource )
- throws IOException
+ throws IOException
{
final InputStream inputStream = resource.read();
if ( inputStream == null )
{
return null;
}
- return loadProperties( inputStream );
- }
-
- /**
- * Saves {@link Properties} to provided {@link OutputStream} and closes the stream.
- */
- public static void storeProperties( final OutputStream outputStream, final Properties properties )
- throws IOException
- {
- try
+ try ( InputStream is = inputStream )
{
- properties.store( outputStream, "Maven Indexer Writer" );
- }
- finally
- {
- outputStream.close();
+ final Properties properties = new Properties();
+ properties.load( is );
+ return properties;
}
}
-
/**
* Saves {@link Properties} to provided {@link WritableResource} and closes the resource.
*/
public static void storeProperties( final WritableResource writableResource, final Properties properties )
- throws IOException
+ throws IOException
{
- try
- {
- storeProperties( writableResource.write(), properties );
- }
- finally
+ try ( OutputStream outputStream = writableResource.write() )
{
- writableResource.close();
+ properties.store( outputStream, "Maven Indexer Writer" );
}
}
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
index 207b076..3580c47 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.OutputStream;
/**
- * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
+ * Maven Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
* updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link
* IndexReader} invocation. Is used by single thread only.
*
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
index 5e2dd32..dace48e 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
@@ -53,7 +53,6 @@ public class CachingResourceHandler
}
public Resource locate( final String name )
- throws IOException
{
if ( notFoundResources.contains( name ) )
{
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
index eb6a393..76ca7fa 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
@@ -19,10 +19,6 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.apache.maven.index.reader.ResourceHandler.Resource;
-import org.junit.Test;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -31,59 +27,62 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link ChunkReader}
*/
public class ChunkReaderTest
- extends TestSupport
+ extends TestSupport
{
@Test
public void simple()
- throws IOException
+ throws IOException
{
- final ChunkReader chunkReader = new ChunkReader( "full", testResourceHandler( "simple" ) //
- .locate( "nexus-maven-repository-index.gz" ).read() );
- final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
- assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ ChunkReader chunkReader = new ChunkReader( "full",
+ resourceHandler.locate( "nexus-maven-repository-index.gz" ).read() ) )
+ {
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ }
}
@Test
public void roundtrip()
- throws IOException
+ throws IOException
{
final Date published;
File tempChunkFile = createTempFile( "nexus-maven-repository-index.gz" );
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ ChunkReader chunkReader = new ChunkReader( "full",
+ resourceHandler.locate( "nexus-maven-repository-index.gz" ).read() );
+ ChunkWriter chunkWriter = new ChunkWriter( chunkReader.getName(), new FileOutputStream( tempChunkFile ),
+ 1, new Date() ) )
{
- final Resource resource = testResourceHandler( "simple" ) //
- .locate( "nexus-maven-repository-index.gz" );
-
- try (ChunkReader chunkReader = new ChunkReader( "full",
- resource.read() ); ChunkWriter chunkWriter = new ChunkWriter(
- chunkReader.getName(), //
- new FileOutputStream( tempChunkFile ), 1, new Date() ))
- {
- chunkWriter.writeChunk( chunkReader.iterator() );
- published = chunkWriter.getTimestamp();
- }
-
+ chunkWriter.writeChunk( chunkReader.iterator() );
+ published = chunkWriter.getTimestamp();
}
- final ChunkReader chunkReader = new ChunkReader( "full", new FileInputStream( tempChunkFile ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( published.getTime() ) );
- final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
- assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
- assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ try ( ChunkReader chunkReader = new ChunkReader( "full", new FileInputStream( tempChunkFile ) ) )
+ {
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( published.getTime() ) );
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 2 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
+ }
}
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
index eaa49e8..2b205df 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
@@ -34,56 +34,64 @@ import java.io.OutputStream;
* by name from specified existing directory.
*/
public class DirectoryResourceHandler
- implements WritableResourceHandler
+ implements WritableResourceHandler
{
- private final File rootDirectory;
+ private final File rootDirectory;
- public DirectoryResourceHandler(final File rootDirectory) {
- if (rootDirectory == null) {
- throw new NullPointerException("null rootDirectory");
+ public DirectoryResourceHandler( final File rootDirectory )
+ {
+ if ( rootDirectory == null )
+ {
+ throw new NullPointerException( "null rootDirectory" );
+ }
+ if ( !rootDirectory.isDirectory() )
+ {
+ throw new IllegalArgumentException( "rootDirectory exists and is not a directory" );
+ }
+ this.rootDirectory = rootDirectory;
}
- if (!rootDirectory.isDirectory()) {
- throw new IllegalArgumentException("rootDirectory exists and is not a directory");
- }
- this.rootDirectory = rootDirectory;
- }
- public File getRootDirectory() {
- return rootDirectory;
- }
+ public WritableResource locate( final String name )
+ {
+ return new FileResource( new File( rootDirectory, name ) );
+ }
- public WritableResource locate(final String name) throws IOException {
- return new FileResource( new File( rootDirectory, name ) );
- }
+ public void close() throws IOException
+ {
+ // nop
+ }
- public void close() throws IOException {
- // nop
- }
+ private static class FileResource
+ implements WritableResource
+ {
+ private final File file;
- private static class FileResource
- implements WritableResource
- {
- private final File file;
+ private FileResource( final File file )
+ {
+ this.file = file;
+ }
- private FileResource(final File file) {
- this.file = file;
- }
+ public InputStream read() throws IOException
+ {
+ try
+ {
+ return new BufferedInputStream( new FileInputStream( file ) );
+ }
+ catch ( FileNotFoundException e )
+ {
+ return null;
+ }
+ }
- public InputStream read() throws IOException {
- try {
- return new BufferedInputStream(new FileInputStream(file));
- } catch (FileNotFoundException e) {
- return null;
- }
- }
-
- public OutputStream write() throws IOException {
- return new BufferedOutputStream(new FileOutputStream(file));
- }
+ public OutputStream write() throws IOException
+ {
+ return new BufferedOutputStream( new FileOutputStream( file ) );
+ }
- public void close() throws IOException {
- // nop
+ public void close() throws IOException
+ {
+ // nop
+ }
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
index e19d949..fe1043b 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
@@ -32,40 +32,46 @@ import java.net.URL;
* handle any advanced cases, like redirects, authentication, etc.
*/
public class HttpResourceHandler
- implements ResourceHandler
+ implements ResourceHandler
{
- private final URI root;
+ private final URI root;
- public HttpResourceHandler(final URL root) throws URISyntaxException {
- if (root == null) {
- throw new NullPointerException("root URL null");
+ public HttpResourceHandler( final URL root ) throws URISyntaxException
+ {
+ if ( root == null )
+ {
+ throw new NullPointerException( "root URL null" );
+ }
+ this.root = root.toURI();
}
- this.root = root.toURI();
- }
- public Resource locate(final String name) throws IOException {
- return new HttpResource(name);
- }
+ public Resource locate( final String name )
+ {
+ return new HttpResource( name );
+ }
- private class HttpResource
- implements Resource
- {
- private final String name;
+ private class HttpResource
+ implements Resource
+ {
+ private final String name;
- private HttpResource(final String name) {
- this.name = name;
- }
+ private HttpResource( final String name )
+ {
+ this.name = name;
+ }
- public InputStream read() throws IOException {
- URL target = root.resolve(name).toURL();
- HttpURLConnection conn = (HttpURLConnection) target.openConnection();
- conn.setRequestMethod("GET");
- conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0");
- return new BufferedInputStream(conn.getInputStream());
+ public InputStream read() throws IOException
+ {
+ URL target = root.resolve( name ).toURL();
+ HttpURLConnection conn = (HttpURLConnection) target.openConnection();
+ conn.setRequestMethod( "GET" );
+ conn.setRequestProperty( "User-Agent", "ASF Maven-Indexer-Reader/1.0" );
+ return new BufferedInputStream( conn.getInputStream() );
+ }
}
- }
- public void close() throws IOException {
- // nop
- }
+ public void close() throws IOException
+ {
+ // nop
+ }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
index cc5c2d2..0fb7867 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
@@ -19,187 +19,215 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.Ignore;
-import org.junit.Test;
-
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
+import java.util.Iterator;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import static com.google.common.collect.Iterables.transform;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Ignore;
+import org.junit.Test;
+
import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.hamcrest.core.IsNot.not;
import static org.hamcrest.core.IsNull.nullValue;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link IndexReader}
*/
public class IndexReaderTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void simple() throws IOException {
- try ( IndexReader indexReader = new IndexReader( null, testResourceHandler( "simple" ) ) )
+ @Test
+ public void simple() throws IOException
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
- for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ try ( IndexReader indexReader = new IndexReader( null, testResourceHandler( "simple" ) ) )
{
- records++;
- }
- }
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ int records = 0;
+
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
+ for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
+ .map( expandFunction )
+ .collect( Collectors.toList() ) )
+ {
+ records++;
+ }
+ }
+ }
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records, equalTo( 5 ) );
+ }
}
- }
- @Test
- public void simpleWithLocal() throws IOException {
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- try ( IndexReader indexReader = new IndexReader( writableResourceHandler, testResourceHandler( "simple" ) ) )
+ @Test
+ public void simpleWithLocal() throws IOException
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
- for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
{
- records++;
- }
- }
+ try ( IndexReader indexReader = new IndexReader( writableResourceHandler,
+ testResourceHandler( "simple" ) ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( 1243533418015L ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(),
+ equalTo( Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ int records = 0;
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
- }
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ assertThat( chunkReader.getTimestamp().getTime(), equalTo( 1243533418015L ) );
+ for ( Record ignored : StreamSupport.stream( chunkReader.spliterator(), false )
+ .map( expandFunction )
+ .collect( Collectors.toList() ) )
+ {
+ records++;
+ }
+ }
+ }
- assertThat(writableResourceHandler.locate("nexus-maven-repository-index.properties").read(), not(nullValue()));
- }
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records, equalTo( 5 ) );
+ }
+ assertThat( writableResourceHandler.locate( "nexus-maven-repository-index.properties" ).read(),
+ not( nullValue() ) );
+ }
+ }
- @Test
- public void roundtrip() throws IOException {
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- Date published;
+ @Test
+ public void roundtrip() throws IOException
{
- final IndexReader indexReader = new IndexReader(
- null,
- testResourceHandler("simple")
- );
- final IndexWriter indexWriter = new IndexWriter(
- writableResourceHandler,
- indexReader.getIndexId(),
- false
- );
- try {
- for (ChunkReader chunkReader : indexReader) {
- indexWriter.writeChunk(chunkReader.iterator());
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ Date published;
+ {
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ IndexReader indexReader = new IndexReader( null, resourceHandler );
+ IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexReader.getIndexId(),
+ false ) )
+ {
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ indexWriter.writeChunk( chunkReader.iterator() );
+ }
+ }
+ indexWriter.close(); // must close to store properties, so next call returns non-null
+ published = indexWriter.getPublishedTimestamp();
+ }
+ }
+
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( published.getTime() ) );
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ AtomicInteger records = new AtomicInteger( 0 );
+
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ chunkReader.forEach( r -> records.incrementAndGet() );
+ }
+ }
+
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records.get(), equalTo( 5 ) );
+ }
}
- }
- finally {
- indexWriter.close();
- published = indexWriter.getPublishedTimestamp();
- indexReader.close();
- }
}
- try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ /**
+ * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your
+ * IP address being banned. You were warned!
+ */
+ @Test
+ @Ignore( "For eyes only" )
+ public void central() throws Exception
{
- assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
- assertThat( indexReader.getPublishedTimestamp().getTime(), equalTo( published.getTime() ) );
- assertThat( indexReader.isIncremental(), equalTo( false ) );
- assertThat( indexReader.getChunkNames(), equalTo( Arrays.asList( "nexus-maven-repository-index.gz" ) ) );
- int chunks = 0;
- int records = 0;
- for ( ChunkReader chunkReader : indexReader )
- {
- chunks++;
- assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
- assertThat( chunkReader.getVersion(), equalTo( 1 ) );
- // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for ( Record record : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) )
+ // local index location, against which we perform incremental updates
+ final File indexDir = createTempDirectory();
+ // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early
+ final File cacheDir = createTempDirectory();
+
+ final PrintWriter writer = new PrintWriter( System.out, true );
+ final WritableResourceHandler local = new DirectoryResourceHandler( indexDir );
+ final CachingResourceHandler remote = new CachingResourceHandler(
+ new DirectoryResourceHandler( cacheDir ),
+ new HttpResourceHandler( new URL( "http://repo1.maven.org/maven2/.index/" ) )
+ );
+ final IndexReader indexReader = new IndexReader( local, remote );
+ try
{
- records++;
- }
- }
+ writer.println( "indexRepoId=" + indexReader.getIndexId() );
+ writer.println( "indexLastPublished=" + indexReader.getPublishedTimestamp() );
+ writer.println( "isIncremental=" + indexReader.isIncremental() );
+ writer.println( "indexRequiredChunkNames=" + indexReader.getChunkNames() );
- assertThat( chunks, equalTo( 1 ) );
- assertThat( records, equalTo( 5 ) );
- }
- }
-
- /**
- * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your
- * IP address being banned. You were warned!
- */
- @Test
- @Ignore("For eyes only")
- public void central() throws Exception {
- // local index location, against which we perform incremental updates
- final File indexDir = createTempDirectory();
- // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early
- final File cacheDir = createTempDirectory();
-
- final PrintWriter writer = new PrintWriter(System.out, true);
- final WritableResourceHandler local = new DirectoryResourceHandler(indexDir);
- final CachingResourceHandler remote = new CachingResourceHandler(
- new DirectoryResourceHandler(cacheDir),
- new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/"))
- );
- final IndexReader indexReader = new IndexReader(local, remote);
- try {
- writer.println("indexRepoId=" + indexReader.getIndexId());
- writer.println("indexLastPublished=" + indexReader.getPublishedTimestamp());
- writer.println("isIncremental=" + indexReader.isIncremental());
- writer.println("indexRequiredChunkNames=" + indexReader.getChunkNames());
- for (ChunkReader chunkReader : indexReader) {
- writer.println("chunkName=" + chunkReader.getName());
- writer.println("chunkVersion=" + chunkReader.getVersion());
- writer.println("chunkPublished=" + chunkReader.getTimestamp());
- writer.println("Chunk stats:");
- Map<Type, Integer> stats = countRecordsByType(chunkReader);
- for (Map.Entry<Type, Integer> entry : stats.entrySet()) {
- writer.println(entry.getKey() + " = " + entry.getValue());
+ Iterator<ChunkReader> chunkReaders = indexReader.iterator();
+ while ( chunkReaders.hasNext() )
+ {
+ try ( ChunkReader chunkReader = chunkReaders.next() )
+ {
+ writer.println( "chunkName=" + chunkReader.getName() );
+ writer.println( "chunkVersion=" + chunkReader.getVersion() );
+ writer.println( "chunkPublished=" + chunkReader.getTimestamp() );
+ writer.println( "Chunk stats:" );
+ Map<Type, Integer> stats = countRecordsByType( chunkReader );
+ for ( Map.Entry<Type, Integer> entry : stats.entrySet() )
+ {
+ writer.println( entry.getKey() + " = " + entry.getValue() );
+ }
+ writer.println( "= = = = = =" );
+ }
+ }
+ }
+ finally
+ {
+ indexReader.close();
+ remote.close();
+ local.close();
}
- writer.println("= = = = = =");
- }
- }
- finally {
- indexReader.close();
- remote.close();
- local.close();
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
index 48aadb2..fd97cbc 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
@@ -19,79 +19,57 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.junit.Test;
-
import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
-import static com.google.common.collect.Iterables.transform;
-import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* UT for {@link IndexWriter}
*/
public class IndexWriterTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void roundtrip() throws IOException {
- IndexReader indexReader;
- IndexWriter indexWriter;
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ @Test
+ public void roundtrip() throws IOException
+ {
+ try ( ResourceHandler resourceHandler = testResourceHandler( "simple" );
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ try ( IndexReader indexReader = new IndexReader( null, resourceHandler );
+ IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexReader.getIndexId(),
+ false ) )
+ {
+ for ( ChunkReader chunkReader : indexReader )
+ {
+ indexWriter.writeChunk( chunkReader.iterator() );
+ }
+ }
- // write it once
- indexReader = new IndexReader(
- null,
- testResourceHandler("simple")
- );
- indexWriter = new IndexWriter(
- writableResourceHandler,
- indexReader.getIndexId(),
- false
- );
- try {
- for (ChunkReader chunkReader : indexReader) {
- indexWriter.writeChunk(chunkReader.iterator());
- }
- }
- finally {
- indexWriter.close();
- indexReader.close();
- }
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getIndexId(), equalTo( "apache-snapshots-local" ) );
+ // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat( indexReader.isIncremental(), equalTo( false ) );
+ assertThat( indexReader.getChunkNames(),
+ equalTo( Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ int chunks = 0;
+ AtomicInteger records = new AtomicInteger( 0 );
+ for ( ChunkReader chunkReader : indexReader )
+ {
+ chunks++;
+ assertThat( chunkReader.getName(), equalTo( "nexus-maven-repository-index.gz" ) );
+ assertThat( chunkReader.getVersion(), equalTo( 1 ) );
+ chunkReader.forEach( r -> records.incrementAndGet() );
+ }
- // read what we wrote out
- indexReader = new IndexReader(
- null,
- writableResourceHandler
- );
- try {
- assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
- // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
- assertThat(indexReader.isIncremental(), equalTo(false));
- assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
- int chunks = 0;
- int records = 0;
- for (ChunkReader chunkReader : indexReader) {
- chunks++;
- assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
- assertThat(chunkReader.getVersion(), equalTo(1));
- // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for (Record record : StreamSupport.stream( chunkReader.spliterator(), false )
- .map( expandFunction )
- .collect( Collectors.toList() ) ) {
- records++;
+ assertThat( chunks, equalTo( 1 ) );
+ assertThat( records.get(), equalTo( 5 ) );
+ }
}
- }
-
- assertThat(chunks, equalTo(1));
- assertThat(records, equalTo(5));
- }
- finally {
- indexReader.close();
}
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
index 72f1040..0f299e5 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
@@ -19,23 +19,22 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
* Test support.
@@ -54,7 +53,7 @@ public class TestSupport
*/
@Before
public void setup()
- throws IOException
+ throws IOException
{
this.tempDir = Files.createTempDirectory( getClass().getSimpleName() + ".temp" ).toFile();
this.directoryResourceHandlers = new ArrayList<>();
@@ -65,7 +64,7 @@ public class TestSupport
*/
@After
public void cleanup()
- throws IOException
+ throws IOException
{
for ( DirectoryResourceHandler directoryResourceHandler : directoryResourceHandlers )
{
@@ -78,7 +77,6 @@ public class TestSupport
* Creates a temp file within {@link #tempDir} with given name.
*/
protected File createTempFile( final String name )
- throws IOException
{
File file = new File( tempDir, name );
file.deleteOnExit();
@@ -89,7 +87,7 @@ public class TestSupport
* Creates a temp directory within {@link #tempDir}.
*/
protected File createTempDirectory()
- throws IOException
+ throws IOException
{
return Files.createTempDirectory( tempDir.toPath(), testName.getMethodName() + "-dir" ).toFile();
}
@@ -98,7 +96,7 @@ public class TestSupport
* Creates an empty {@link DirectoryResourceHandler}.
*/
protected WritableResourceHandler createWritableResourceHandler()
- throws IOException
+ throws IOException
{
DirectoryResourceHandler result = new DirectoryResourceHandler( createTempDirectory() );
directoryResourceHandlers.add( result );
@@ -110,7 +108,6 @@ public class TestSupport
* name.
*/
protected ResourceHandler testResourceHandler( final String name )
- throws IOException
{
DirectoryResourceHandler result = new DirectoryResourceHandler( new File( "src/test/resources/" + name ) );
directoryResourceHandlers.add( result );
@@ -121,7 +118,7 @@ public class TestSupport
* Consumes {@link ChunkReader} and creates a map "by type" with records.
*/
protected Map<Type, List<Record>> loadRecordsByType( final ChunkReader chunkReader )
- throws IOException
+ throws IOException
{
HashMap<Type, List<Record>> stat = new HashMap<>();
try
@@ -150,7 +147,7 @@ public class TestSupport
* Consumes {@link ChunkReader} and creates a map "by type" with record type counts.
*/
protected Map<Type, Integer> countRecordsByType( final ChunkReader chunkReader )
- throws IOException
+ throws IOException
{
HashMap<Type, Integer> stat = new HashMap<>();
try
@@ -173,35 +170,4 @@ public class TestSupport
}
return stat;
}
-
- /**
- * Delete recursively.
- */
- private static boolean delete( final File file )
- {
- if ( file == null )
- {
- return false;
- }
- if ( !file.exists() )
- {
- return true;
- }
- if ( file.isDirectory() )
- {
- String[] list = file.list();
- if ( list != null )
- {
- for ( String s : list )
- {
- File entry = new File( file, s );
- if ( !delete( entry ) )
- {
- return false;
- }
- }
- }
- }
- return file.delete();
- }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
index 2e6912b..8ae5eef 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
@@ -19,76 +19,80 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.Type;
-
import java.util.Map;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import org.apache.maven.index.reader.Record.Type;
+
import static com.google.common.collect.Iterables.concat;
import static java.util.Collections.singletonList;
-import static org.apache.maven.index.reader.Utils.*;
+import static org.apache.maven.index.reader.Utils.allGroups;
+import static org.apache.maven.index.reader.Utils.descriptor;
+import static org.apache.maven.index.reader.Utils.rootGroup;
+import static org.apache.maven.index.reader.Utils.rootGroups;
/**
* Helpers to transform records from one to another representation, and, some helpers for publishing using Guava.
*/
public final class TestUtils
{
- private TestUtils() {
- // nothing
- }
+ private TestUtils()
+ {
+ // nothing
+ }
- private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor();
+ private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor();
- private static final RecordExpander RECORD_EXPANDER = new RecordExpander();
+ private static final RecordExpander RECORD_EXPANDER = new RecordExpander();
- public static Function<Record, Map<String, String>> compactFunction = RECORD_COMPACTOR::apply;
+ public static Function<Record, Map<String, String>> compactFunction = RECORD_COMPACTOR::apply;
- public static Function<Map<String, String>, Record> expandFunction = RECORD_EXPANDER::apply;
+ public static Function<Map<String, String>, Record> expandFunction = RECORD_EXPANDER::apply;
- /**
- * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
- * all the caller is needed to provide {@link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
- * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
- * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
- * records.
- */
- public static Iterable<Record> decorate(final Iterable<Record> iterable,
- final String repoId)
- {
- final TreeSet<String> allGroupsSet = new TreeSet<>();
- final TreeSet<String> rootGroupsSet = new TreeSet<>();
- return StreamSupport.stream(
- concat( singletonList( descriptor( repoId ) ), iterable, singletonList( allGroups( allGroupsSet ) ),
- // placeholder, will be recreated at the end with proper content
- singletonList( rootGroups( rootGroupsSet ) )
- // placeholder, will be recreated at the end with proper content
- ).spliterator(), false ).map( rec ->
+ /**
+ * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
+ * all the caller is needed to provide {@link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
+ * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
+ * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
+ * records.
+ */
+ public static Iterable<Record> decorate( final Iterable<Record> iterable,
+ final String repoId )
{
- if ( Type.DESCRIPTOR == rec.getType() )
- {
- return rec;
- }
- else if ( Type.ALL_GROUPS == rec.getType() )
- {
- return allGroups( allGroupsSet );
- }
- else if ( Type.ROOT_GROUPS == rec.getType() )
- {
- return rootGroups( rootGroupsSet );
- }
- else
- {
- final String groupId = rec.get( Record.GROUP_ID );
- if ( groupId != null )
+ final TreeSet<String> allGroupsSet = new TreeSet<>();
+ final TreeSet<String> rootGroupsSet = new TreeSet<>();
+ return StreamSupport.stream(
+ concat( singletonList( descriptor( repoId ) ), iterable, singletonList( allGroups( allGroupsSet ) ),
+ // placeholder, will be recreated at the end with proper content
+ singletonList( rootGroups( rootGroupsSet ) )
+ // placeholder, will be recreated at the end with proper content
+ ).spliterator(), false ).map( rec ->
{
- allGroupsSet.add( groupId );
- rootGroupsSet.add( rootGroup( groupId ) );
- }
- return rec;
- }
- } ).collect( Collectors.toList() );
- }
+ if ( Type.DESCRIPTOR == rec.getType() )
+ {
+ return rec;
+ }
+ else if ( Type.ALL_GROUPS == rec.getType() )
+ {
+ return allGroups( allGroupsSet );
+ }
+ else if ( Type.ROOT_GROUPS == rec.getType() )
+ {
+ return rootGroups( rootGroupsSet );
+ }
+ else
+ {
+ final String groupId = rec.get( Record.GROUP_ID );
+ if ( groupId != null )
+ {
+ allGroupsSet.add( groupId );
+ rootGroupsSet.add( rootGroup( groupId ) );
+ }
+ return rec;
+ }
+ } ).collect( Collectors.toList() );
+ }
}
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
index 1ea6155..4817b7f 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
@@ -19,83 +19,86 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import org.apache.maven.index.reader.Record.EntryKey;
-import org.apache.maven.index.reader.Record.Type;
-import org.junit.Test;
-
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import static com.google.common.collect.Iterables.transform;
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
import static org.apache.maven.index.reader.TestUtils.compactFunction;
import static org.apache.maven.index.reader.TestUtils.decorate;
import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
/**
- * UT for {@link RecordCompactor} and {@linl RecordExpander}.
+ * UT for {@link RecordCompactor} and {@link RecordExpander}.
*/
public class TransformTest
- extends TestSupport
+ extends TestSupport
{
- @Test
- public void decorateAndTransform() throws IOException {
- final String indexId = "test";
- final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache"));
- final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo"));
- final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar"));
+ @Test
+ public void decorateAndTransform() throws IOException
+ {
+ final String indexId = "test";
+ final Record r1 = new Record( Type.ARTIFACT_ADD, artifactMap( "org.apache" ) );
+ final Record r2 = new Record( Type.ARTIFACT_ADD, artifactMap( "org.foo" ) );
+ final Record r3 = new Record( Type.ARTIFACT_ADD, artifactMap( "com.bar" ) );
- Iterable<Map<String, String>> iterable = StreamSupport.stream(
- decorate( Arrays.asList( r1, r2, r3 ), indexId ).spliterator(), false )
- .map( compactFunction )
- .collect( Collectors.toList() );
+ Iterable<Map<String, String>> iterable = StreamSupport.stream(
+ decorate( Arrays.asList( r1, r2, r3 ), indexId ).spliterator(), false )
+ .map( compactFunction )
+ .collect( Collectors.toList() );
- WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
- try {
- IndexWriter indexWriter = new IndexWriter(
- writableResourceHandler,
- indexId,
- false
- );
- indexWriter.writeChunk(iterable.iterator());
- indexWriter.close();
- }
- finally {
- writableResourceHandler.close();
- }
+ try ( WritableResourceHandler writableResourceHandler = createWritableResourceHandler() )
+ {
+ try ( IndexWriter indexWriter = new IndexWriter( writableResourceHandler, indexId, false ) )
+ {
+ indexWriter.writeChunk( iterable.iterator() );
+ }
- IndexReader indexReader = new IndexReader(null, writableResourceHandler);
- assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
- ChunkReader chunkReader = indexReader.iterator().next();
- final Map<Type, List<Record>> recordTypes = loadRecordsByType(chunkReader);
- assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
- assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(3));
- assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+ try ( IndexReader indexReader = new IndexReader( null, writableResourceHandler ) )
+ {
+ assertThat( indexReader.getChunkNames(), equalTo(
+ Collections.singletonList( "nexus-maven-repository-index.gz" ) ) );
+ try ( ChunkReader chunkReader = indexReader.iterator().next() )
+ {
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType( chunkReader );
+ assertThat( recordTypes.get( Type.DESCRIPTOR ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).size(), equalTo( 1 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_ADD ).size(), equalTo( 3 ) );
+ assertThat( recordTypes.get( Type.ARTIFACT_REMOVE ), nullValue() );
- assertThat(recordTypes.get(Type.ROOT_GROUPS).get(0).get(Record.ROOT_GROUPS), equalTo(new String[] {"com","org"}));
- assertThat(recordTypes.get(Type.ALL_GROUPS).get(0).get(Record.ALL_GROUPS), equalTo(new String[] {"com.bar", "org.apache", "org.foo"}));
- }
+ assertThat( recordTypes.get( Type.ROOT_GROUPS ).get( 0 ).get( Record.ROOT_GROUPS ),
+ equalTo( new String[] {"com", "org"} ) );
+ assertThat( recordTypes.get( Type.ALL_GROUPS ).get( 0 ).get( Record.ALL_GROUPS ),
+ equalTo( new String[] {"com.bar", "org.apache", "org.foo"} ) );
+ }
+ }
+ }
+ }
- private Map<EntryKey, Object> artifactMap(final String groupId) {
- final HashMap<EntryKey, Object> result = new HashMap<>();
- result.put(Record.GROUP_ID, groupId);
- result.put(Record.ARTIFACT_ID, "artifact");
- result.put(Record.VERSION, "1.0");
- result.put(Record.PACKAGING, "jar");
- result.put(Record.FILE_MODIFIED, System.currentTimeMillis());
- result.put(Record.FILE_SIZE, 123L);
- result.put(Record.FILE_EXTENSION, "jar");
- result.put(Record.HAS_SOURCES, Boolean.FALSE);
- result.put(Record.HAS_JAVADOC, Boolean.FALSE);
- result.put(Record.HAS_SIGNATURE, Boolean.FALSE);
- return result;
- }
+ private Map<EntryKey, Object> artifactMap( final String groupId )
+ {
+ final HashMap<EntryKey, Object> result = new HashMap<>();
+ result.put( Record.GROUP_ID, groupId );
+ result.put( Record.ARTIFACT_ID, "artifact" );
+ result.put( Record.VERSION, "1.0" );
+ result.put( Record.PACKAGING, "jar" );
+ result.put( Record.FILE_MODIFIED, System.currentTimeMillis() );
+ result.put( Record.FILE_SIZE, 123L );
+ result.put( Record.FILE_EXTENSION, "jar" );
+ result.put( Record.HAS_SOURCES, Boolean.FALSE );
+ result.put( Record.HAS_JAVADOC, Boolean.FALSE );
+ result.put( Record.HAS_SIGNATURE, Boolean.FALSE );
+ return result;
+ }
}