You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2022/11/28 07:22:10 UTC
[maven-indexer] branch master updated: [MINDEXER-176] Clean up silos (#273)
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-indexer.git
The following commit(s) were added to refs/heads/master by this push:
new c2a438a [MINDEXER-176] Clean up silos (#273)
c2a438a is described below
commit c2a438a6b57458045a0481fbeb8b23f1d91d5cb6
Author: Tamas Cservenak <ta...@cservenak.net>
AuthorDate: Mon Nov 28 08:22:05 2022 +0100
[MINDEXER-176] Clean up silos (#273)
The MT updater did not clean up silos after done.
---
https://issues.apache.org/jira/browse/MINDEXER-176
---
.../maven/index/updater/IndexDataReader.java | 38 ++++++++++++++++------
1 file changed, 28 insertions(+), 10 deletions(-)
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
index aac2ef4..f826a56 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
@@ -23,6 +23,7 @@ import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.EOFException;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UTFDataFormatException;
@@ -46,6 +47,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.context.IndexUtils;
@@ -171,11 +173,15 @@ public class IndexDataReader
ExecutorService executorService = Executors.newFixedThreadPool( threads );
ArrayList<Exception> errors = new ArrayList<>();
- ArrayList<IndexWriter> silos = new ArrayList<>( threads );
+ ArrayList<FSDirectory> siloDirectories = new ArrayList<>( threads );
+ ArrayList<IndexWriter> siloWriters = new ArrayList<>( threads );
+ LOGGER.debug( "Creating {} silo writer threads...", threads );
for ( int i = 0; i < threads; i++ )
{
final int silo = i;
- silos.add( tempWriter( "silo" + i ) );
+ FSDirectory siloDirectory = tempDirectory( "silo" + i );
+ siloDirectories.add( siloDirectory );
+ siloWriters.add( tempWriter( siloDirectory ) );
executorService.execute( () ->
{
LOGGER.debug( "Starting thread {}", Thread.currentThread().getName() );
@@ -190,7 +196,7 @@ public class IndexDataReader
{
break;
}
- addToIndex( doc, context, silos.get( silo ), rootGroups, allGroups );
+ addToIndex( doc, context, siloWriters.get( silo ), rootGroups, allGroups );
}
catch ( InterruptedException | IOException e )
{
@@ -206,6 +212,7 @@ public class IndexDataReader
} );
}
+ LOGGER.debug( "Loading up documents into silos" );
try
{
Document doc;
@@ -244,14 +251,25 @@ public class IndexDataReader
IndexUtils.updateTimestamp( w.getDirectory(), date );
}
- LOGGER.debug( "Merging silos..." );
- for ( IndexWriter silo : silos )
+ LOGGER.debug( "Closing silo writers..." );
+ for ( IndexWriter siloWriter : siloWriters )
{
- IndexUtils.close( silo );
- w.addIndexes( silo.getDirectory() );
+ siloWriter.commit();
+ siloWriter.close();
}
- LOGGER.debug( "Merged silos..." );
+ LOGGER.debug( "Merging silo directories..." );
+ w.addIndexes( siloDirectories.toArray( new Directory[0] ) );
+
+ LOGGER.debug( "Cleanup of silo directories..." );
+ for ( FSDirectory siloDirectory : siloDirectories )
+ {
+ File dir = siloDirectory.getDirectory().toFile();
+ siloDirectory.close();
+ IndexUtils.delete( dir );
+ }
+
+ LOGGER.debug( "Finalizing..." );
w.commit();
IndexDataReadResult result = new IndexDataReadResult();
@@ -269,11 +287,11 @@ public class IndexDataReader
return FSDirectory.open( Files.createTempDirectory( name + ".dir" ) );
}
- private IndexWriter tempWriter( final String name ) throws IOException
+ private IndexWriter tempWriter( final FSDirectory directory ) throws IOException
{
IndexWriterConfig config = new IndexWriterConfig( new NexusAnalyzer() );
config.setUseCompoundFile( false );
- return new NexusIndexWriter( tempDirectory( name ), config );
+ return new NexusIndexWriter( directory, config );
}
private void addToIndex( final Document doc, final IndexingContext context, final IndexWriter indexWriter,