You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by ma...@apache.org on 2022/05/21 20:53:32 UTC
[netbeans] branch master updated: Fix maven groupId completion
This is an automated email from the ASF dual-hosted git repository.
matthiasblaesing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new d13544ef96 Fix maven groupId completion
new 7cb6c63cc5 Merge pull request #4136 from matthiasblaesing/maven_group_list
d13544ef96 is described below
commit d13544ef964b8e7bbe7c2707c41247cf5089ddda
Author: Matthias Bläsing <mb...@doppel-helix.eu>
AuthorDate: Fri May 20 20:15:27 2022 +0200
Fix maven groupId completion
MavenIndexer dropped the caching of the allGroups and rootGroups
properties in the index:
https://github.com/apache/maven-indexer/commit/c231db602b5cea9ed20eba617de5a5a846ef5faa
After the index is loaded there are two options:
a) rebuild the groups list from the index
b) cache the group list after indexing
NetBeans expects the index to be readily be available and thus option
a is not an option (rebuilding the group list runs over the full index
and is in the minute range). As there are only a few entrypoints that
modify the index caching the group list looks like a viable option.
Closes: #4102
---
.../maven/indexer/NexusRepositoryIndexerImpl.java | 123 ++++++++++++++++++---
1 file changed, 105 insertions(+), 18 deletions(-)
diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
index aeb8ac648e..3545f7c23f 100644
--- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
+++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
@@ -32,6 +32,7 @@ import java.net.URI;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
@@ -131,6 +132,11 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
private static final Logger LOGGER = Logger.getLogger(NexusRepositoryIndexerImpl.class.getName());
+ private static final String GROUP_CACHE_ALL_PREFIX = "nb-groupcache-all-v1"; // NOI18N
+ private static final String GROUP_CACHE_ALL_SUFFIX = "txt"; // NOI18N
+ private static final String GROUP_CACHE_ROOT_PREFIX = "nb-groupcache-root-v1"; // NOI18N
+ private static final String GROUP_CACHE_ROOT_SUFFIX = "txt"; // NOI18N
+
private PlexusContainer embedder;
private Indexer indexer;
private org.apache.maven.index.Scanner scanner;
@@ -257,14 +263,50 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
}
//TODO try to experiment with the non-forced version of the context addition
- public IndexingContext addIndexingContextForced(String id, String repositoryId, File repository,
- File indexDirectory, String repositoryUrl, String indexUpdateUrl, List<? extends IndexCreator> indexers) throws IOException {
-
- IndexingContext context = indexer.createIndexingContext(id, repositoryId, repository, indexDirectory, repositoryUrl, indexUpdateUrl, true, true, indexers);
+ private IndexingContext addIndexingContextForced(
+ RepositoryInfo repo,
+ List<? extends IndexCreator> indexers
+ ) throws IOException {
+
+ IndexingContext context = indexer.createIndexingContext(
+ /* id */ repo.getId(),
+ /* repositoryId */ repo.getId(),
+ /* repository */ repo.isLocal() ? new File(repo.getRepositoryPath()) : null,
+ /* indexDirectory */ getIndexDirectory(repo),
+ /* repositoryUrl */ repo.isRemoteDownloadable() ? repo.getRepositoryUrl() : null,
+ /* indexUpdateUrl */ repo.isRemoteDownloadable() ? repo.getIndexUpdateUrl() : null,
+ /* searchable */ true,
+ /* reclaim */ true,
+ /* indexers */ indexers);
+
+ // The allGroups and rootGroups properties of the IndexingContext are
+ // not persistent anymore, so need to be save outside the context
+ boolean needGroupCacheRebuild = false;
+ try {
+ context.setAllGroups(Files.readAllLines(getAllGroupCacheFile(repo)));
+ } catch (IOException ex) {
+ needGroupCacheRebuild = true;
+ }
+ try {
+ context.setRootGroups(Files.readAllLines(getRootGroupCacheFile(repo)));
+ } catch (IOException ex) {
+ needGroupCacheRebuild = true;
+ }
+ // At least one of the group caches could not be loaded, so rebuild it
+ if(needGroupCacheRebuild) {
+ RP.submit(() -> {
+ try {
+ context.rebuildGroups();
+ storeGroupCache(repo, context);
+ } catch (IOException ex) {
+ LOGGER.log(Level.WARNING, "Failed to store group caches for repo: " + repo.getId(), ex);
+ }
+ });
+ }
indexingContexts.put(context.getId(), context);
return context;
}
-
+
public void removeIndexingContext(IndexingContext context, boolean deleteFiles) throws IOException {
if (indexingContexts.remove(context.getId()) != null ) {
indexer.closeIndexingContext( context, deleteFiles );
@@ -296,7 +338,6 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
}
}
LOGGER.log(Level.FINE, "Loading Context: {0}", info.getId());
- File loc = new File(getDefaultIndexLocation(), info.getId()); // index folder
List<IndexCreator> creators = new ArrayList<>();
try {
@@ -318,17 +359,10 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
creators.add(new NotifyingIndexCreator());
}
try {
- addIndexingContextForced(
- info.getId(), // context id
- info.getId(), // repository id
- info.isLocal() ? new File(info.getRepositoryPath()) : null, // repository folder
- loc,
- info.isRemoteDownloadable() ? info.getRepositoryUrl() : null, // repositoryUrl
- info.isRemoteDownloadable() ? indexUpdateUrl : null,
- creators);
+ addIndexingContextForced(info, creators);
LOGGER.log(Level.FINE, "using index creators: {0}", creators);
} catch (IOException ex) {
- LOGGER.log(Level.INFO, "Found a broken index at " + loc + " with loaded contexts " + getIndexingContexts().keySet(), ex);
+ LOGGER.log(Level.INFO, "Found a broken index at " + getIndexDirectory(info) + " with loaded contexts " + getIndexingContexts().keySet(), ex);
break LOAD;
}
}
@@ -353,7 +387,7 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
}
}
- File loc = new File(getDefaultIndexLocation(), info.getId()); // index folder
+ File loc = getIndexDirectory(info); // index folder
if (!indexExists(loc.toPath())) {
index = true;
LOGGER.log(Level.FINER, "Index Not Available: {0} at: {1}", new Object[]{info.getId(), loc.getAbsolutePath()});
@@ -496,7 +530,10 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
nic.start(listener);
}
try {
+ Files.deleteIfExists(getAllGroupCacheFile(repo));
+ Files.deleteIfExists(getRootGroupCacheFile(repo));
remoteIndexUpdater.fetchAndUpdateIndex(iur);
+ storeGroupCache(repo, indexingContext);
} catch (IllegalArgumentException ex) {
// This exception is raised from the maven-indexer.
// maven-indexer supported two formats:
@@ -529,7 +566,11 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
} else {
RepositoryIndexerListener repoListener = new RepositoryIndexerListener(indexingContext);
try {
+ // Ensure no stale cache files are left
+ Files.deleteIfExists(getAllGroupCacheFile(repo));
+ Files.deleteIfExists(getRootGroupCacheFile(repo));
scan(indexingContext, null, repoListener, updateLocal);
+ storeGroupCache(repo, indexingContext);
} finally {
repoListener.close();
}
@@ -770,6 +811,7 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
}
try {
indexer.addArtifactsToIndex(artifactContexts, indexingContext);
+ storeGroupCache(repo, indexingContext);
} catch (ZipError err) {
LOGGER.log(Level.INFO, "#230581 concurrent access to local repository file. Skipping..", err);
}
@@ -824,6 +866,7 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
if (pom.exists()) {
//TODO batch removal??
indexer.deleteArtifactsFromIndex(Collections.singleton(contextProducer.getArtifactContext(indexingContext, pom)), indexingContext);
+ storeGroupCache(repo, indexingContext);
}
return null;
});
@@ -1487,7 +1530,7 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
public List<RepositoryInfo> getLoaded(final List<RepositoryInfo> repos) {
final List<RepositoryInfo> toRet = new ArrayList<>(repos.size());
for (final RepositoryInfo repo : repos) {
- File loc = new File(getDefaultIndexLocation(), repo.getId()); // index folder
+ File loc = getIndexDirectory(repo); // index folder
if (indexExists(loc.toPath())) {
toRet.add(repo);
}
@@ -1633,5 +1676,49 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
return new WagonHelper.WagonFetcher(wagon, listener, authenticationInfo, proxyInfo);
}
}
-
+
+ private File getIndexDirectory(final RepositoryInfo info) {
+ return new File(getDefaultIndexLocation(), info.getId());
+ }
+
+ private Path getAllGroupCacheFile(final RepositoryInfo info) {
+ return new File(getIndexDirectory(info), GROUP_CACHE_ALL_PREFIX + "." + GROUP_CACHE_ALL_SUFFIX).toPath();
+ }
+
+ private Path getRootGroupCacheFile(final RepositoryInfo info) {
+ return new File(getIndexDirectory(info), GROUP_CACHE_ROOT_PREFIX + "." + GROUP_CACHE_ROOT_SUFFIX).toPath();
+ }
+
+ private void storeGroupCache(RepositoryInfo repoInfo, IndexingContext ic) throws IOException {
+ // MINDEXER-157: After import all groups and root groups are inverted in the IndexingContext
+ Set<String> allGroups = ic.getAllGroups();
+ Set<String> rootGroups = ic.getRootGroups();
+ if(rootGroups.size() > allGroups.size()) {
+ // Inversion - allGroups should always have the same size as rootGroups or be larger
+ ic.setAllGroups(rootGroups);
+ ic.setRootGroups(allGroups);
+ } else if (rootGroups.size() == allGroups.size()) {
+ // Inversion if any groupId in rootGroups contains a dot (so is not a root id)
+ boolean inversion = rootGroups.stream().anyMatch(s -> s.contains("."));
+ if (inversion) {
+ ic.setAllGroups(rootGroups);
+ ic.setRootGroups(allGroups);
+ }
+ }
+
+ Path indexDir = getIndexDirectory(repoInfo).toPath();
+ Path tempAllCache = Files.createTempFile(indexDir, GROUP_CACHE_ALL_PREFIX, GROUP_CACHE_ALL_SUFFIX);
+ Path tempRootCache = Files.createTempFile(indexDir, GROUP_CACHE_ROOT_PREFIX, GROUP_CACHE_ROOT_SUFFIX);
+ try {
+ Files.write(tempAllCache, ic.getAllGroups());
+ Files.move(tempAllCache, getAllGroupCacheFile(repoInfo), StandardCopyOption.REPLACE_EXISTING);
+
+ Files.write(tempRootCache, ic.getRootGroups());
+ Files.move(tempRootCache, getRootGroupCacheFile(repoInfo), StandardCopyOption.REPLACE_EXISTING);
+ } finally {
+ Files.deleteIfExists(tempAllCache);
+ Files.deleteIfExists(tempRootCache);
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@netbeans.apache.org
For additional commands, e-mail: commits-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists