You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by ro...@apache.org on 2019/05/08 16:16:08 UTC
[sling-whiteboard] 04/09: Improvements:
This is an automated email from the ASF dual-hosted git repository.
rombert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sling-whiteboard.git
commit 277182112fd1c99aceb325240340548a288d8cd3
Author: Robert Munteanu <ro...@apache.org>
AuthorDate: Mon May 6 16:46:43 2019 +0200
Improvements:
- Move the bulk of the code to an IndexQuerier class
- Use the available API to check if sources are present
---
.../msra/impl/{Main.java => IndexQuerier.java} | 110 ++++-------
.../org/apache/sling/tooling/msra/impl/Main.java | 210 +--------------------
2 files changed, 38 insertions(+), 282 deletions(-)
diff --git a/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java b/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/IndexQuerier.java
similarity index 72%
copy from maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java
copy to maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/IndexQuerier.java
index 2165203..d5245c0 100644
--- a/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java
+++ b/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/IndexQuerier.java
@@ -18,6 +18,8 @@
*/
package org.apache.sling.tooling.msra.impl;
+import static org.apache.maven.index.ArtifactAvailability.PRESENT;
+
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@@ -28,9 +30,9 @@ import java.util.Date;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.ArtifactInfoGroup;
import org.apache.maven.index.GroupedSearchRequest;
@@ -59,19 +61,17 @@ import org.codehaus.plexus.component.repository.exception.ComponentLookupExcepti
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class Main {
+public class IndexQuerier {
- public static void main(String[] args) throws Exception {
-
- if ( args.length == 0 ) {
- throw new IllegalArgumentException("Please specify groupId to look for, e.g. org.apache*");
- }
-
- Main main = new Main(args[0]);
- main.loadIndex();
- main.querySlingApiArtifacts();
+ private static ArtifactInfo extract(ArtifactInfoGroup group) {
+
+ return group.getArtifactInfos()
+ .stream()
+ .filter(a -> a.getClassifier() == null)
+ .findFirst()
+ .orElse(null);
}
-
+
private static final File OUT_DIR = new File("out");
private final PlexusContainer plexusContainer;
@@ -83,19 +83,15 @@ public class Main {
private final Wagon httpWagon;
private IndexingContext centralContext;
-
+
private final Logger logger = LoggerFactory.getLogger(getClass());
- private final String queryGroupId;
- public Main(String queryGroupId) throws PlexusContainerException, ComponentLookupException {
- // here we create Plexus container, the Maven default IoC container
- // Plexus falls outside of MI scope, just accept the fact that
- // MI is a Plexus component ;)
- // If needed more info, ask on Maven Users list or Plexus Users list
- // google is your friend!
- final DefaultContainerConfiguration config = new DefaultContainerConfiguration();
+ public IndexQuerier() throws PlexusContainerException, ComponentLookupException {
+
+ DefaultContainerConfiguration config = new DefaultContainerConfiguration();
config.setClassPathScanning(PlexusConstants.SCANNING_INDEX);
+
this.plexusContainer = new DefaultPlexusContainer(config);
// lookup the indexer components from plexus
@@ -103,8 +99,6 @@ public class Main {
this.indexUpdater = plexusContainer.lookup(IndexUpdater.class);
// lookup wagon used to remotely fetch index
this.httpWagon = plexusContainer.lookup(Wagon.class, "https");
- this.queryGroupId = queryGroupId;
-
}
public void loadIndex() throws ComponentLookupException, IOException {
@@ -155,22 +149,19 @@ public class Main {
} else if (updateResult.getTimestamp().equals(centralContextCurrentTimestamp)) {
logger.info("No update needed, index is up to date!");
} else {
- logger.info("Incremental update happened, change covered {} - {} period.",
- centralContextCurrentTimestamp , updateResult.getTimestamp());
+ logger.info("Incremental update happened, change covered {} - {} period.", centralContextCurrentTimestamp,
+ updateResult.getTimestamp());
}
}
-
- public void querySlingApiArtifacts() throws IOException {
-
+
+ public void querySourceArtifacts(String queryGroupId) throws IOException {
+
Query groupId = indexer.constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression(queryGroupId));
Query packaging = indexer.constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("pom"));
- BooleanQuery bq = new BooleanQuery.Builder()
- .add(groupId, Occur.MUST)
- .add(packaging, Occur.MUST_NOT)
- .build();
+ BooleanQuery bq = new BooleanQuery.Builder().add(groupId, Occur.MUST).add(packaging, Occur.MUST_NOT).build();
- searchAndDump( indexer, "all " + queryGroupId + " artifacts", bq );
+ searchAndDump(indexer, "all " + queryGroupId + " artifacts", bq);
}
public void searchAndDump(Indexer nexusIndexer, String descr, Query q) throws IOException {
@@ -179,59 +170,28 @@ public class Main {
GroupedSearchResponse response = nexusIndexer
.searchGrouped(new GroupedSearchRequest(q, new GAGrouping(), centralContext));
- try ( FileOutputStream fos = new FileOutputStream(new File(OUT_DIR, "results.csv"));
- BufferedOutputStream bos = new BufferedOutputStream(fos);
+ try (FileOutputStream fos = new FileOutputStream(new File(OUT_DIR, "results.csv"));
+ BufferedOutputStream bos = new BufferedOutputStream(fos);
OutputStreamWriter w = new OutputStreamWriter(bos)) {
-
+
response.getResults().values()
.stream()
- .map( Main::extract )
- .filter( Objects::nonNull )
- .forEach( aws -> writeLine(w, aws));
-
+ .map(IndexQuerier::extract)
+ .filter(Objects::nonNull)
+ .forEach(ai -> writeLine(w, ai));
+
logger.info("Total hits: {}", response.getTotalHitsCount());
}
-
+
logger.info("Data written under {}", OUT_DIR);
}
- private void writeLine(OutputStreamWriter w, ArtifactWithSource aws) {
+ private void writeLine(OutputStreamWriter w, ArtifactInfo aws) {
try {
- w.write(aws.main.getGroupId() + ',' +
- aws.main.getArtifactId()+ ',' +
- aws.main.getVersion()+ ',' +
- (aws.source != null ? '1' : '0') + '\n');
+ w.write(aws.getGroupId() + ',' + aws.getArtifactId() + ',' + aws.getVersion() + ','
+ + (aws.getSourcesExists() == PRESENT ? '1' : '0') + '\n');
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
- private static ArtifactWithSource extract(ArtifactInfoGroup group) {
-
- ArtifactWithSource aws = new ArtifactWithSource();
-
- aws.main = group.getArtifactInfos()
- .stream()
- .filter( a -> a.getClassifier() == null)
- .findFirst()
- .orElse(null);
-
- if ( aws.main == null )
- return null;
-
- // TODO - need constant for 'sources'
- aws.source = group.getArtifactInfos().stream()
- .filter( a -> a.getVersion().equals(aws.main.getVersion()) && "sources".equals(a.getClassifier()) )
- .findFirst()
- .orElse(null);
-
- return aws;
- }
-
- static class ArtifactWithSource {
-
- private ArtifactInfo main;
-
- private ArtifactInfo source;
- }
}
diff --git a/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java b/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java
index 2165203..c588ba5 100644
--- a/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java
+++ b/maven-sources-report-aggregator/src/main/java/org/apache/sling/tooling/msra/impl/Main.java
@@ -18,47 +18,6 @@
*/
package org.apache.sling.tooling.msra.impl;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Query;
-import org.apache.maven.index.ArtifactInfo;
-import org.apache.maven.index.ArtifactInfoGroup;
-import org.apache.maven.index.GroupedSearchRequest;
-import org.apache.maven.index.GroupedSearchResponse;
-import org.apache.maven.index.Indexer;
-import org.apache.maven.index.MAVEN;
-import org.apache.maven.index.context.IndexCreator;
-import org.apache.maven.index.context.IndexingContext;
-import org.apache.maven.index.expr.SourcedSearchExpression;
-import org.apache.maven.index.search.grouping.GAGrouping;
-import org.apache.maven.index.updater.IndexUpdateRequest;
-import org.apache.maven.index.updater.IndexUpdateResult;
-import org.apache.maven.index.updater.IndexUpdater;
-import org.apache.maven.index.updater.ResourceFetcher;
-import org.apache.maven.index.updater.WagonHelper;
-import org.apache.maven.wagon.Wagon;
-import org.apache.maven.wagon.events.TransferEvent;
-import org.apache.maven.wagon.events.TransferListener;
-import org.apache.maven.wagon.observers.AbstractTransferListener;
-import org.codehaus.plexus.DefaultContainerConfiguration;
-import org.codehaus.plexus.DefaultPlexusContainer;
-import org.codehaus.plexus.PlexusConstants;
-import org.codehaus.plexus.PlexusContainer;
-import org.codehaus.plexus.PlexusContainerException;
-import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class Main {
public static void main(String[] args) throws Exception {
@@ -67,171 +26,8 @@ public class Main {
throw new IllegalArgumentException("Please specify groupId to look for, e.g. org.apache*");
}
- Main main = new Main(args[0]);
- main.loadIndex();
- main.querySlingApiArtifacts();
- }
-
- private static final File OUT_DIR = new File("out");
-
- private final PlexusContainer plexusContainer;
-
- private final Indexer indexer;
-
- private final IndexUpdater indexUpdater;
-
- private final Wagon httpWagon;
-
- private IndexingContext centralContext;
-
- private final Logger logger = LoggerFactory.getLogger(getClass());
-
- private final String queryGroupId;
-
- public Main(String queryGroupId) throws PlexusContainerException, ComponentLookupException {
- // here we create Plexus container, the Maven default IoC container
- // Plexus falls outside of MI scope, just accept the fact that
- // MI is a Plexus component ;)
- // If needed more info, ask on Maven Users list or Plexus Users list
- // google is your friend!
- final DefaultContainerConfiguration config = new DefaultContainerConfiguration();
- config.setClassPathScanning(PlexusConstants.SCANNING_INDEX);
- this.plexusContainer = new DefaultPlexusContainer(config);
-
- // lookup the indexer components from plexus
- this.indexer = plexusContainer.lookup(Indexer.class);
- this.indexUpdater = plexusContainer.lookup(IndexUpdater.class);
- // lookup wagon used to remotely fetch index
- this.httpWagon = plexusContainer.lookup(Wagon.class, "https");
- this.queryGroupId = queryGroupId;
-
- }
-
- public void loadIndex() throws ComponentLookupException, IOException {
- // Files where local cache is (if any) and Lucene Index should be located
- File centralLocalCache = new File(OUT_DIR, "central-cache");
- File centralIndexDir = new File(OUT_DIR, "central-index");
-
- // Creators we want to use (search for fields it defines)
- List<IndexCreator> indexers = new ArrayList<>();
- indexers.add(plexusContainer.lookup(IndexCreator.class, "min"));
- indexers.add(plexusContainer.lookup(IndexCreator.class, "jarContent"));
- indexers.add(plexusContainer.lookup(IndexCreator.class, "maven-plugin"));
-
- // Create context for central repository index
- centralContext = indexer.createIndexingContext("central-context", "central", centralLocalCache, centralIndexDir,
- "https://repo1.maven.org/maven2", null, true, true, indexers);
-
- // Update the index (incremental update will happen if this is not 1st run and
- // files are not deleted)
- // This whole block below should not be executed on every app start, but rather
- // controlled by some configuration
- // since this block will always emit at least one HTTP GET. Central indexes are
- // updated once a week, but
- // other index sources might have different index publishing frequency.
- // Preferred frequency is once a week.
- logger.info("Updating Index. This might take a while on first run, so please be patient.");
- // Create ResourceFetcher implementation to be used with IndexUpdateRequest
- // Here, we use Wagon based one as shorthand, but all we need is a
- // ResourceFetcher implementation
- TransferListener listener = new AbstractTransferListener() {
- @Override
- public void transferStarted(TransferEvent transferEvent) {
- logger.info("Downloading {}", transferEvent.getResource().getName());
- }
-
- @Override
- public void transferCompleted(TransferEvent transferEvent) {
- logger.info("Done downloading {}", transferEvent.getResource().getName());
- }
- };
- ResourceFetcher resourceFetcher = new WagonHelper.WagonFetcher(httpWagon, listener, null, null);
-
- Date centralContextCurrentTimestamp = centralContext.getTimestamp();
- IndexUpdateRequest updateRequest = new IndexUpdateRequest(centralContext, resourceFetcher);
- IndexUpdateResult updateResult = indexUpdater.fetchAndUpdateIndex(updateRequest);
- if (updateResult.isFullUpdate()) {
- logger.info("Full update happened!");
- } else if (updateResult.getTimestamp().equals(centralContextCurrentTimestamp)) {
- logger.info("No update needed, index is up to date!");
- } else {
- logger.info("Incremental update happened, change covered {} - {} period.",
- centralContextCurrentTimestamp , updateResult.getTimestamp());
- }
- }
-
- public void querySlingApiArtifacts() throws IOException {
-
- Query groupId = indexer.constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression(queryGroupId));
- Query packaging = indexer.constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("pom"));
-
- BooleanQuery bq = new BooleanQuery.Builder()
- .add(groupId, Occur.MUST)
- .add(packaging, Occur.MUST_NOT)
- .build();
-
- searchAndDump( indexer, "all " + queryGroupId + " artifacts", bq );
- }
-
- public void searchAndDump(Indexer nexusIndexer, String descr, Query q) throws IOException {
- logger.info("Searching for {}", descr);
-
- GroupedSearchResponse response = nexusIndexer
- .searchGrouped(new GroupedSearchRequest(q, new GAGrouping(), centralContext));
-
- try ( FileOutputStream fos = new FileOutputStream(new File(OUT_DIR, "results.csv"));
- BufferedOutputStream bos = new BufferedOutputStream(fos);
- OutputStreamWriter w = new OutputStreamWriter(bos)) {
-
- response.getResults().values()
- .stream()
- .map( Main::extract )
- .filter( Objects::nonNull )
- .forEach( aws -> writeLine(w, aws));
-
- logger.info("Total hits: {}", response.getTotalHitsCount());
- }
-
- logger.info("Data written under {}", OUT_DIR);
- }
-
- private void writeLine(OutputStreamWriter w, ArtifactWithSource aws) {
- try {
- w.write(aws.main.getGroupId() + ',' +
- aws.main.getArtifactId()+ ',' +
- aws.main.getVersion()+ ',' +
- (aws.source != null ? '1' : '0') + '\n');
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static ArtifactWithSource extract(ArtifactInfoGroup group) {
-
- ArtifactWithSource aws = new ArtifactWithSource();
-
- aws.main = group.getArtifactInfos()
- .stream()
- .filter( a -> a.getClassifier() == null)
- .findFirst()
- .orElse(null);
-
- if ( aws.main == null )
- return null;
-
- // TODO - need constant for 'sources'
- aws.source = group.getArtifactInfos().stream()
- .filter( a -> a.getVersion().equals(aws.main.getVersion()) && "sources".equals(a.getClassifier()) )
- .findFirst()
- .orElse(null);
-
- return aws;
- }
-
- static class ArtifactWithSource {
-
- private ArtifactInfo main;
-
- private ArtifactInfo source;
+ IndexQuerier q = new IndexQuerier();
+ q.loadIndex();
+ q.querySourceArtifacts(args[0]);
}
}