You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2022/04/25 11:44:23 UTC
[maven-indexer] 01/01: [MINDEXER-146] Fix issues reported by Lucene 9
This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch MINDEXER-146-backport-fixes
in repository https://gitbox.apache.org/repos/asf/maven-indexer.git
commit 698cda1e8712879706ecd34e80cbd04f05de69e2
Author: Tamas Cservenak <ta...@cservenak.net>
AuthorDate: Mon Apr 25 13:42:49 2022 +0200
[MINDEXER-146] Fix issues reported by Lucene 9
There were several "sloppy" spots especially about indexing
options and norms (field type options), that are now
fixed. Also, replaced uses of deprecated RAMDirectory with
new ByteBuffersDirectory, that affects only test code.
---
.../index/context/DefaultIndexingContext.java | 5 +++--
.../maven/index/updater/IndexDataReader.java | 3 +--
.../maven/index/updater/IndexDataWriter.java | 5 +++--
.../maven/index/AbstractNexusIndexerTest.java | 8 ++------
.../index/ConcurrentUseWithMergedContextTest.java | 6 +++---
.../apache/maven/index/DuplicateSearchTest.java | 6 +++---
.../maven/index/Nexus13NexusIndexerTest.java | 4 ++--
.../maven/index/Nexus3177HitLimitChecks.java | 4 ++--
.../org/apache/maven/index/NexusIndexerTest.java | 5 +++--
.../index/context/TrackingLockFactoryTest.java | 9 +++++----
.../index/updater/DefaultIndexUpdaterTest.java | 22 +++++++++++-----------
.../apache/maven/index/updater/IndexDataTest.java | 12 +++++-------
12 files changed, 43 insertions(+), 46 deletions(-)
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java b/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
index fcf1441..4e3a4ce 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
@@ -369,7 +369,8 @@ public class DefaultIndexingContext
hdr.add( new Field( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, IndexerField.KEYWORD_STORED ) );
- hdr.add( new StoredField( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId() ) );
+ hdr.add( new StoredField( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(),
+ IndexerField.KEYWORD_STORED ) );
IndexWriter w = getIndexWriter();
@@ -879,7 +880,7 @@ public class DefaultIndexingContext
{
final Document groupDoc = new Document();
groupDoc.add( new Field( field, fieldValue, IndexerField.KEYWORD_STORED ) );
- groupDoc.add( new StoredField( listField, ArtifactInfo.lst2str( groups ) ) );
+ groupDoc.add( new StoredField( listField, ArtifactInfo.lst2str( groups ), IndexerField.KEYWORD_STORED ) );
return groupDoc;
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
index d8cc804..d670cf0 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
@@ -184,9 +184,8 @@ public class IndexDataReader
{
boolean tokenized = ( flags & IndexDataWriter.F_TOKENIZED ) > 0;
fieldType.setTokenized( tokenized );
- fieldType.setOmitNorms( !tokenized );
- fieldType.setIndexOptions( IndexOptions.DOCS_AND_FREQS_AND_POSITIONS );
}
+ fieldType.setIndexOptions( IndexOptions.DOCS_AND_FREQS_AND_POSITIONS );
fieldType.setStored( ( flags & IndexDataWriter.F_STORED ) > 0 );
String name = dis.readUTF();
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
index bd759fe..1be10d5 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
@@ -32,6 +32,7 @@ import java.util.Set;
import java.util.zip.GZIPOutputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
@@ -124,7 +125,7 @@ public class IndexDataWriter
List<IndexableField> allGroupsFields = new ArrayList<>( 2 );
allGroupsFields.add( new Field( ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE,
IndexerField.KEYWORD_STORED ) );
- allGroupsFields.add( new Field( ArtifactInfo.ALL_GROUPS_LIST, ArtifactInfo.lst2str( allGroups ),
+ allGroupsFields.add( new StoredField( ArtifactInfo.ALL_GROUPS_LIST, ArtifactInfo.lst2str( allGroups ),
IndexerField.KEYWORD_STORED ) );
writeDocumentFields( allGroupsFields );
}
@@ -133,7 +134,7 @@ public class IndexDataWriter
List<IndexableField> rootGroupsFields = new ArrayList<>( 2 );
rootGroupsFields.add( new Field( ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE,
IndexerField.KEYWORD_STORED ) );
- rootGroupsFields.add( new Field( ArtifactInfo.ROOT_GROUPS_LIST, ArtifactInfo.lst2str( rootGroups ),
+ rootGroupsFields.add( new StoredField( ArtifactInfo.ROOT_GROUPS_LIST, ArtifactInfo.lst2str( rootGroups ),
IndexerField.KEYWORD_STORED ) );
writeDocumentFields( rootGroupsFields );
}
diff --git a/indexer-core/src/test/java/org/apache/maven/index/AbstractNexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/AbstractNexusIndexerTest.java
index 16c1d1c..8df4d1a 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/AbstractNexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/AbstractNexusIndexerTest.java
@@ -23,12 +23,8 @@ import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.maven.index.ArtifactInfo;
-import org.apache.maven.index.FlatSearchRequest;
-import org.apache.maven.index.FlatSearchResponse;
-import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexingContext;
public abstract class AbstractNexusIndexerTest
@@ -36,7 +32,7 @@ public abstract class AbstractNexusIndexerTest
{
protected NexusIndexer nexusIndexer;
- protected Directory indexDir = new RAMDirectory();
+ protected Directory indexDir = new ByteBuffersDirectory();
protected IndexingContext context;
diff --git a/indexer-core/src/test/java/org/apache/maven/index/ConcurrentUseWithMergedContextTest.java b/indexer-core/src/test/java/org/apache/maven/index/ConcurrentUseWithMergedContextTest.java
index 8db7c9e..859ea3a 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/ConcurrentUseWithMergedContextTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/ConcurrentUseWithMergedContextTest.java
@@ -21,8 +21,8 @@ package org.apache.maven.index;
import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.context.IndexingContext;
/**
@@ -34,11 +34,11 @@ import org.apache.maven.index.context.IndexingContext;
public class ConcurrentUseWithMergedContextTest
extends ConcurrentUseTest
{
- protected Directory indexDir1 = new RAMDirectory();
+ protected Directory indexDir1 = new ByteBuffersDirectory();
protected IndexingContext context1;
- protected Directory indexDir2 = new RAMDirectory();
+ protected Directory indexDir2 = new ByteBuffersDirectory();
protected IndexingContext context2;
diff --git a/indexer-core/src/test/java/org/apache/maven/index/DuplicateSearchTest.java b/indexer-core/src/test/java/org/apache/maven/index/DuplicateSearchTest.java
index a97764e..4cab9d7 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/DuplicateSearchTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/DuplicateSearchTest.java
@@ -27,8 +27,8 @@ import java.util.HashSet;
import junit.framework.Assert;
import org.apache.lucene.search.Query;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.expr.SourcedSearchExpression;
@@ -39,11 +39,11 @@ public class DuplicateSearchTest
protected IndexingContext context1;
- protected Directory contextDir1 = new RAMDirectory();
+ protected Directory contextDir1 = new ByteBuffersDirectory();
protected IndexingContext context2;
- protected Directory contextDir2 = new RAMDirectory();
+ protected Directory contextDir2 = new ByteBuffersDirectory();
@Override
protected void prepareNexusIndexer( NexusIndexer nexusIndexer )
diff --git a/indexer-core/src/test/java/org/apache/maven/index/Nexus13NexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/Nexus13NexusIndexerTest.java
index 76acc5b..be128e1 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/Nexus13NexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/Nexus13NexusIndexerTest.java
@@ -31,8 +31,8 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.packer.IndexPackingRequest;
@@ -121,7 +121,7 @@ public class Nexus13NexusIndexerTest
Thread.sleep( 1000L );
- Directory indexDir = new RAMDirectory();
+ Directory indexDir = new ByteBuffersDirectory();
IndexingContext newContext =
nexusIndexer.addIndexingContext( "test-new", "nexus-13", null, indexDir, null, null, DEFAULT_CREATORS );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/Nexus3177HitLimitChecks.java b/indexer-core/src/test/java/org/apache/maven/index/Nexus3177HitLimitChecks.java
index b8700d1..ab30553 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/Nexus3177HitLimitChecks.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/Nexus3177HitLimitChecks.java
@@ -24,8 +24,8 @@ import java.util.Set;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.context.IndexingContext;
public class Nexus3177HitLimitChecks
@@ -33,7 +33,7 @@ public class Nexus3177HitLimitChecks
{
protected File repo = new File( getBasedir(), "src/test/repo" );
- protected Directory secondIndexDir = new RAMDirectory();
+ protected Directory secondIndexDir = new ByteBuffersDirectory();
protected IndexingContext secondContext;
diff --git a/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
index a221747..d08f951 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
@@ -40,7 +40,8 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
import org.apache.maven.index.context.IndexCreator;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.MergedIndexingContext;
@@ -582,7 +583,7 @@ public class NexusIndexerTest
indexer.removeIndexingContext( context, false );
- RAMDirectory newDirectory = new RAMDirectory();
+ Directory newDirectory = new ByteBuffersDirectory();
IndexingContext newContext = indexer.addIndexingContext( indexId, //
repositoryId, repository, newDirectory, repositoryUrl, null, indexCreators );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java b/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java
index 5590f6c..d35741e 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java
@@ -20,9 +20,10 @@ package org.apache.maven.index.context;
import java.io.IOException;
import java.util.Set;
+
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -39,7 +40,7 @@ public class TrackingLockFactoryTest {
@Test
public void testLockUnlock() throws IOException {
final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
- final RAMDirectory ram = new RAMDirectory(lf);
+ final ByteBuffersDirectory ram = new ByteBuffersDirectory(lf);
final Lock foo = ram.obtainLock("foo");
final Lock bar = ram.obtainLock("bar");
bar.close();
@@ -50,7 +51,7 @@ public class TrackingLockFactoryTest {
@Test
public void testLockLocked() throws IOException {
final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
- final RAMDirectory ram = new RAMDirectory(lf);
+ final ByteBuffersDirectory ram = new ByteBuffersDirectory(lf);
final Lock foo = ram.obtainLock("foo");
boolean thrownLOFE = false;
try {
@@ -68,7 +69,7 @@ public class TrackingLockFactoryTest {
@Test
public void testEmmittedLocks() throws IOException {
final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
- final RAMDirectory ram = new RAMDirectory(lf);
+ final ByteBuffersDirectory ram = new ByteBuffersDirectory(lf);
final Lock l1 = ram.obtainLock("l1");
final Lock l2 = ram.obtainLock("l2");
final Lock l3 = ram.obtainLock("l3");
diff --git a/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java b/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
index 4deea97..527e3f1 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
@@ -33,9 +33,9 @@ import java.util.Set;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.FlatSearchRequest;
import org.apache.maven.index.FlatSearchResponse;
@@ -74,7 +74,7 @@ public class DefaultIndexUpdaterTest
// updated index
- Directory tempIndexDirectory = new RAMDirectory();
+ Directory tempIndexDirectory = new ByteBuffersDirectory();
IndexingContext tempContext =
indexer.addIndexingContext( repositoryId + "temp", repositoryId, null, tempIndexDirectory, repositoryUrl,
@@ -95,7 +95,7 @@ public class DefaultIndexUpdaterTest
// A change in RAMDirectory and Directory behavior in general: it will copy the Index files ONLY
// So we must make sure that timestamp file is transferred correctly.
- RAMDirectory tempDir2 = new RAMDirectory();
+ ByteBuffersDirectory tempDir2 = new ByteBuffersDirectory();
IndexUtils.copyDirectory( tempContext.getIndexDirectory(), tempDir2 );
Date newIndexTimestamp = tempContext.getTimestamp();
@@ -129,7 +129,7 @@ public class DefaultIndexUpdaterTest
// updated index
{
- Directory tempIndexDirectory = new RAMDirectory();
+ Directory tempIndexDirectory = new ByteBuffersDirectory();
IndexingContext tempContext =
indexer.addIndexingContext( repositoryId + "temp", repositoryId, null, tempIndexDirectory,
@@ -149,7 +149,7 @@ public class DefaultIndexUpdaterTest
Collection<ArtifactInfo> tempContent = tempResponse.getResults();
assertEquals( tempContent.toString(), 3, tempContent.size() );
- RAMDirectory tempDir2 = new RAMDirectory();
+ ByteBuffersDirectory tempDir2 = new ByteBuffersDirectory();
for (String file : tempContext.getIndexDirectory().listAll())
{
tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
@@ -178,7 +178,7 @@ public class DefaultIndexUpdaterTest
context );
{
- Directory tempIndexDirectory = new RAMDirectory();
+ Directory tempIndexDirectory = new ByteBuffersDirectory();
IndexingContext tempContext =
indexer.addIndexingContext( repositoryId + "temp", repositoryId, null, tempIndexDirectory,
@@ -196,7 +196,7 @@ public class DefaultIndexUpdaterTest
indexer.deleteArtifactFromIndex(
createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.4", null ), tempContext );
- RAMDirectory tempDir2 = new RAMDirectory();
+ ByteBuffersDirectory tempDir2 = new ByteBuffersDirectory();
for (String file : tempContext.getIndexDirectory().listAll())
{
tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
@@ -219,14 +219,14 @@ public class DefaultIndexUpdaterTest
throws Exception
{
File repo1 = new File( getBasedir(), "src/test/nexus-658" );
- Directory indexDir1 = new RAMDirectory();
+ Directory indexDir1 = new ByteBuffersDirectory();
IndexingContext context1 =
indexer.addIndexingContext( "nexus-658", "nexus-658", repo1, indexDir1, null, null, DEFAULT_CREATORS );
indexer.scan( context1 );
File repo2 = new File( getBasedir(), "src/test/nexus-13" );
- Directory indexDir2 = new RAMDirectory();
+ Directory indexDir2 = new ByteBuffersDirectory();
IndexingContext context2 =
indexer.addIndexingContext( "nexus-13", "nexus-13", repo2, indexDir2, null, null, DEFAULT_CREATORS );
@@ -259,7 +259,7 @@ public class DefaultIndexUpdaterTest
context );
{
- Directory tempIndexDirectory = new RAMDirectory();
+ Directory tempIndexDirectory = new ByteBuffersDirectory();
IndexingContext tempContext =
indexer.addIndexingContext( repositoryId + "temp", repositoryId, null, tempIndexDirectory,
@@ -274,7 +274,7 @@ public class DefaultIndexUpdaterTest
indexer.addArtifactToIndex(
createArtifactContext( repositoryId, "org.slf4j.foo", "jcl104-over-slf4j", "1.4.2", null ), context );
- RAMDirectory tempDir2 = new RAMDirectory();
+ ByteBuffersDirectory tempDir2 = new ByteBuffersDirectory();
for (String file : tempContext.getIndexDirectory().listAll())
{
tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
diff --git a/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java b/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
index 03500ea..434030e 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
@@ -32,14 +32,12 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.maven.index.AbstractRepoNexusIndexerTest;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.context.IndexUtils;
-import org.apache.maven.index.updater.DefaultIndexUpdater;
-import org.apache.maven.index.updater.IndexDataWriter;
/**
* @author Eugene Kuleshov
@@ -53,7 +51,7 @@ public class IndexDataTest
protected void prepareNexusIndexer( NexusIndexer nexusIndexer )
throws Exception
{
- indexDir = new RAMDirectory();
+ indexDir = new ByteBuffersDirectory();
context =
nexusIndexer.addIndexingContext( "test-default", "test", repo, indexDir, null, null, DEFAULT_CREATORS );
@@ -82,7 +80,7 @@ public class IndexDataTest
ByteArrayInputStream is = new ByteArrayInputStream( bos.toByteArray() );
- newDir = new RAMDirectory();
+ newDir = new ByteBuffersDirectory();
Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context ).getTimestamp();
@@ -94,7 +92,7 @@ public class IndexDataTest
public void testEmptyContext()
throws Exception
{
- indexDir = new RAMDirectory();
+ indexDir = new ByteBuffersDirectory();
context =
nexusIndexer.addIndexingContext( "test-default", "test", repo, indexDir, null, null, DEFAULT_CREATORS );
@@ -119,7 +117,7 @@ public class IndexDataTest
ByteArrayInputStream is = new ByteArrayInputStream( bos.toByteArray() );
- newDir = new RAMDirectory();
+ newDir = new ByteBuffersDirectory();
Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context ).getTimestamp();