You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2017/01/29 14:43:37 UTC
asterixdb git commit: Cache Knowledge of CDDL Source Jars on Central
Repository: asterixdb
Updated Branches:
refs/heads/master 60e7f12b4 -> 0f3bf37ff
Cache Knowledge of CDDL Source Jars on Central
Avoid downloading source code jars (for CDDL licensed dependendencies)
repeatedly, by persisting the (non-)existence in the local repo.
Change-Id: I961c36d9d64990043598b062bdb542cfb084972a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1472
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
BAD: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/0f3bf37f
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/0f3bf37f
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/0f3bf37f
Branch: refs/heads/master
Commit: 0f3bf37ff40f5b98b49b859c2c91ff5495975d4b
Parents: 60e7f12
Author: Michael Blow <mb...@apache.org>
Authored: Wed Jan 25 21:30:04 2017 -0500
Committer: Michael Blow <mb...@apache.org>
Committed: Sun Jan 29 06:43:11 2017 -0800
----------------------------------------------------------------------
.../hyracks/maven/license/GenerateFileMojo.java | 221 +--------------
.../hyracks/maven/license/LicenseMojo.java | 11 +-
.../maven/license/SourcePointerResolver.java | 274 +++++++++++++++++++
3 files changed, 285 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f3bf37f/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index b604f18..262979a 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -22,11 +22,8 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
-import java.lang.Override;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashMap;
@@ -34,7 +31,6 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -53,31 +49,17 @@ import freemarker.cache.FileTemplateLoader;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hyracks.maven.license.freemarker.IndentDirective;
import org.apache.hyracks.maven.license.freemarker.LoadFileDirective;
import org.apache.hyracks.maven.license.project.LicensedProjects;
import org.apache.hyracks.maven.license.project.Project;
-import org.apache.maven.artifact.Artifact;
-import org.apache.maven.artifact.DefaultArtifact;
-import org.apache.maven.artifact.metadata.ArtifactMetadata;
-import org.apache.maven.artifact.repository.ArtifactRepository;
-import org.apache.maven.artifact.repository.ArtifactRepositoryPolicy;
-import org.apache.maven.artifact.repository.Authentication;
-import org.apache.maven.artifact.repository.DefaultRepositoryRequest;
-import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout;
-import org.apache.maven.artifact.repository.layout.DefaultRepositoryLayout;
-import org.apache.maven.artifact.resolver.ArtifactResolutionRequest;
-import org.apache.maven.artifact.resolver.ArtifactResolutionResult;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
-import org.apache.maven.project.MavenProject;
import org.apache.maven.project.ProjectBuildingException;
-import org.apache.maven.repository.Proxy;
@Mojo(name = "generate",
requiresProject = true,
@@ -128,7 +110,7 @@ public class GenerateFileMojo extends LicenseMojo {
resolveLicenseFiles();
rebuildLicenseContentProjectMap();
combineCommonGavs();
- collectSourceAssemblies();
+ SourcePointerResolver.execute(this);
persistLicenseMap();
buildNoticeProjectMap();
generateFiles();
@@ -137,58 +119,6 @@ public class GenerateFileMojo extends LicenseMojo {
}
}
- private void collectSourceAssemblies() throws ProjectBuildingException, IOException {
- try (StubArtifactRepository stubRepo = new StubArtifactRepository()) {
- DefaultRepositoryRequest rr = new DefaultRepositoryRequest();
- rr.setLocalRepository(stubRepo);
- ArtifactRepository central = getCentralRepository();
- rr.setRemoteRepositories(Collections.singletonList(central));
- ArtifactResolutionRequest request = new ArtifactResolutionRequest(rr);
- for (LicensedProjects lp : licenseMap.values()) {
- if (lp.getLicense().getDisplayName() != null
- && lp.getLicense().getDisplayName().toLowerCase().contains("cddl")) {
- ensureCDDLSourcesPointer(lp.getProjects(), central, request);
- }
- }
- }
- }
-
- private void ensureCDDLSourcesPointer(Collection<Project> projects, ArtifactRepository central,
- ArtifactResolutionRequest request) throws ProjectBuildingException {
- for (Project p : projects) {
- if (p.getSourcePointer() != null) {
- continue;
- }
- getLog().debug("finding sources for artifact: " + p);
- Artifact sourcesArtifact = new DefaultArtifact(p.getGroupId(), p.getArtifactId(),
- p.getVersion(), Artifact.SCOPE_COMPILE, "jar", "sources", null);
- MavenProject mavenProject = resolveDependency(sourcesArtifact);
- sourcesArtifact.setArtifactHandler(mavenProject.getArtifact().getArtifactHandler());
-
- request.setArtifact(sourcesArtifact);
- ArtifactResolutionResult result = artifactResolver.resolve(request);
- getLog().debug("result: " + result);
- StringBuilder noticeBuilder = new StringBuilder("You may obtain ");
- noticeBuilder.append(p.getName()).append(" in Source Code form code here:\n");
- if (result.isSuccess()) {
- noticeBuilder.append(central.getUrl()).append("/").append(central.pathOf(sourcesArtifact));
- } else {
- getLog().warn("Unable to find sources in 'central' for " + p + ", falling back to project url: "
- + p.getUrl());
- noticeBuilder.append(p.getUrl() != null ? p.getUrl() : "MISSING SOURCE POINTER");
- }
- p.setSourcePointer(noticeBuilder.toString());
- }
- }
-
- private ArtifactRepository getCentralRepository() {
- for (ArtifactRepository repo : session.getRequest().getRemoteRepositories()) {
- if ("central".equals(repo.getId())) {
- return repo;
- }
- }
- throw new IllegalStateException("Unable to find 'central' remote repository!");
- }
private void resolveLicenseContent() throws IOException {
Set<LicenseSpec> licenseSpecs = new HashSet<>();
@@ -441,154 +371,5 @@ public class GenerateFileMojo extends LicenseMojo {
}
return matches;
}
-
- private static class StubArtifactRepository implements ArtifactRepository, AutoCloseable {
- private static final Random random = new Random();
- private final File tempDir;
- private final ArtifactRepositoryLayout layout;
-
- public StubArtifactRepository() {
- String tmpDir = System.getProperty("java.io.tmpdir", "/tmp");
- this.tempDir = new File(tmpDir, "repo" + random.nextInt());
- this.layout = new DefaultRepositoryLayout();
- }
-
- @Override
- public ArtifactRepositoryLayout getLayout() {
- return layout;
- }
-
- @Override
- public String pathOf(Artifact artifact) {
- return this.layout.pathOf(artifact);
- }
-
- @Override
- public String getBasedir() {
- return tempDir.toString();
- }
-
- @Override
- public void close() throws IOException {
- FileUtils.deleteDirectory(tempDir);
-
- }
-
- @Override
- public String pathOfRemoteRepositoryMetadata(ArtifactMetadata artifactMetadata) {
- return null;
- }
-
- @Override
- public String pathOfLocalRepositoryMetadata(ArtifactMetadata artifactMetadata,
- ArtifactRepository artifactRepository) {
- return null;
- }
-
- @Override
- public String getUrl() {
- return null;
- }
-
- @Override
- public void setUrl(String s) {
- // unused
- }
-
- @Override
- public String getProtocol() {
- return null;
- }
-
- @Override
- public String getId() {
- return "stub";
- }
-
- @Override
- public void setId(String s) {
- // unused
- }
-
- @Override
- public ArtifactRepositoryPolicy getSnapshots() {
- return null;
- }
-
- @Override
- public void setSnapshotUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
- // unused
- }
-
- @Override
- public ArtifactRepositoryPolicy getReleases() {
- return null;
- }
-
- @Override
- public void setReleaseUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
- // unused
- }
-
- @Override
- public void setLayout(ArtifactRepositoryLayout artifactRepositoryLayout) {
- // unused
- }
-
- @Override
- public String getKey() {
- return null;
- }
-
- @Override
- public boolean isUniqueVersion() {
- return false;
- }
-
- @Override
- public boolean isBlacklisted() {
- return false;
- }
-
- @Override
- public void setBlacklisted(boolean b) {
- // unused
- }
-
- @Override
- public Artifact find(Artifact artifact) {
- return null;
- }
-
- @Override
- public List<String> findVersions(Artifact artifact) {
- return Collections.emptyList();
- }
-
- @Override
- public boolean isProjectAware() {
- return false;
- }
-
- @Override
- public void setAuthentication(Authentication authentication) {
- // unused
- }
-
- @Override
- public Authentication getAuthentication() {
- return null;
- }
-
- @Override
- public void setProxy(Proxy proxy) {
- // unused
- }
-
- @Override
- public Proxy getProxy() {
- return null;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f3bf37f/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
index 929b7d7..75c5bed 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
@@ -40,6 +40,7 @@ import org.apache.hyracks.maven.license.project.LicensedProjects;
import org.apache.hyracks.maven.license.project.Project;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.resolver.ArtifactResolver;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.model.License;
import org.apache.maven.model.Model;
@@ -91,7 +92,7 @@ public abstract class LicenseMojo extends AbstractMojo {
protected MavenSession session;
@Component
- protected org.apache.maven.artifact.resolver.ArtifactResolver artifactResolver;
+ protected ArtifactResolver artifactResolver;
@Parameter ( required = true )
private String location;
@@ -345,5 +346,13 @@ public abstract class LicenseMojo extends AbstractMojo {
}
return locations;
}
+
+ public MavenSession getSession() {
+ return session;
+ }
+
+ public ArtifactResolver getArtifactResolver() {
+ return artifactResolver;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0f3bf37f/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/SourcePointerResolver.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/SourcePointerResolver.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/SourcePointerResolver.java
new file mode 100644
index 0000000..c713b08
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/SourcePointerResolver.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.maven.license;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.maven.license.project.LicensedProjects;
+import org.apache.hyracks.maven.license.project.Project;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.DefaultArtifact;
+import org.apache.maven.artifact.metadata.ArtifactMetadata;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.repository.ArtifactRepositoryPolicy;
+import org.apache.maven.artifact.repository.Authentication;
+import org.apache.maven.artifact.repository.DefaultRepositoryRequest;
+import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout;
+import org.apache.maven.artifact.repository.layout.DefaultRepositoryLayout;
+import org.apache.maven.artifact.resolver.ArtifactResolutionRequest;
+import org.apache.maven.artifact.resolver.ArtifactResolutionResult;
+import org.apache.maven.project.MavenProject;
+import org.apache.maven.project.ProjectBuildingException;
+import org.apache.maven.repository.Proxy;
+
+public class SourcePointerResolver {
+
+ private final GenerateFileMojo mojo;
+
+ private SourcePointerResolver(GenerateFileMojo mojo) {
+ this.mojo = mojo;
+ }
+
+ public static void execute(GenerateFileMojo mojo) throws ProjectBuildingException, IOException {
+ SourcePointerResolver instance = new SourcePointerResolver(mojo);
+ instance.collectSourcePointers();
+ }
+
+ private ArtifactRepository getCentralRepository() {
+ for (ArtifactRepository repo : mojo.getSession().getRequest().getRemoteRepositories()) {
+ if ("central".equals(repo.getId())) {
+ return repo;
+ }
+ }
+ throw new IllegalStateException("Unable to find 'central' remote repository!");
+ }
+
+ private void collectSourcePointers() throws ProjectBuildingException, IOException {
+ try (StubArtifactRepository stubRepo = new StubArtifactRepository()) {
+ DefaultRepositoryRequest rr = new DefaultRepositoryRequest();
+ rr.setLocalRepository(stubRepo);
+ ArtifactRepository central = getCentralRepository();
+ rr.setRemoteRepositories(Collections.singletonList(central));
+ ArtifactResolutionRequest request = new ArtifactResolutionRequest(rr);
+ for (LicensedProjects lp : mojo.getLicenseMap().values()) {
+ if (lp.getLicense().getDisplayName() != null
+ && lp.getLicense().getDisplayName().toLowerCase().contains("cddl")) {
+ ensureCDDLSourcesPointer(lp.getProjects(), central, request);
+ }
+ }
+ }
+ }
+
+ private void ensureCDDLSourcesPointer(Collection<Project> projects, ArtifactRepository central,
+ ArtifactResolutionRequest request)
+ throws ProjectBuildingException, IOException {
+ for (Project p : projects) {
+ if (p.getSourcePointer() != null) {
+ continue;
+ }
+ mojo.getLog().debug("finding sources for artifact: " + p);
+ Artifact sourcesArtifact = new DefaultArtifact(p.getGroupId(), p.getArtifactId(),
+ p.getVersion(), Artifact.SCOPE_COMPILE, "jar", "sources", null);
+ MavenProject mavenProject = mojo.resolveDependency(sourcesArtifact);
+ sourcesArtifact.setArtifactHandler(mavenProject.getArtifact().getArtifactHandler());
+ final ArtifactRepository localRepo = mojo.getSession().getLocalRepository();
+ final File marker = new File(localRepo.getBasedir(), localRepo.pathOf(sourcesArtifact) + ".oncentral");
+ final File antimarker = new File(localRepo.getBasedir(), localRepo.pathOf(sourcesArtifact) + ".nocentral");
+ boolean onCentral;
+ if (marker.exists() || antimarker.exists()) {
+ onCentral = marker.exists();
+ } else {
+ request.setArtifact(sourcesArtifact);
+ ArtifactResolutionResult result = mojo.getArtifactResolver().resolve(request);
+ mojo.getLog().debug("result: " + result);
+ onCentral = result.isSuccess();
+ if (onCentral) {
+ FileUtils.touch(marker);
+ } else {
+ FileUtils.touch(antimarker);
+ }
+ }
+ StringBuilder noticeBuilder = new StringBuilder("You may obtain ");
+ noticeBuilder.append(p.getName()).append(" in Source Code form code here:\n");
+ if (onCentral) {
+ noticeBuilder.append(central.getUrl()).append("/").append(central.pathOf(sourcesArtifact));
+ } else {
+ mojo.getLog().warn("Unable to find sources in 'central' for " + p + ", falling back to project url: "
+ + p.getUrl());
+ noticeBuilder.append(p.getUrl() != null ? p.getUrl() : "MISSING SOURCE POINTER");
+ }
+ p.setSourcePointer(noticeBuilder.toString());
+ }
+ }
+
+ private static class StubArtifactRepository implements ArtifactRepository, AutoCloseable {
+ private static final Random random = new Random();
+ private final File tempDir;
+ private final ArtifactRepositoryLayout layout;
+
+ public StubArtifactRepository() {
+ String tmpDir = System.getProperty("java.io.tmpdir", "/tmp");
+ this.tempDir = new File(tmpDir, "repo" + random.nextInt());
+ this.layout = new DefaultRepositoryLayout();
+ }
+
+ @java.lang.Override
+ public ArtifactRepositoryLayout getLayout() {
+ return layout;
+ }
+
+ @java.lang.Override
+ public String pathOf(Artifact artifact) {
+ return this.layout.pathOf(artifact);
+ }
+
+ @java.lang.Override
+ public String getBasedir() {
+ return tempDir.toString();
+ }
+
+ @java.lang.Override
+ public void close() throws IOException {
+ FileUtils.deleteDirectory(tempDir);
+
+ }
+
+ @java.lang.Override
+ public String pathOfRemoteRepositoryMetadata(ArtifactMetadata artifactMetadata) {
+ return null;
+ }
+
+ @java.lang.Override
+ public String pathOfLocalRepositoryMetadata(ArtifactMetadata artifactMetadata,
+ ArtifactRepository artifactRepository) {
+ return null;
+ }
+
+ @java.lang.Override
+ public String getUrl() {
+ return null;
+ }
+
+ @java.lang.Override
+ public void setUrl(String s) {
+ // unused
+ }
+
+ @java.lang.Override
+ public String getProtocol() {
+ return null;
+ }
+
+ @java.lang.Override
+ public String getId() {
+ return "stub";
+ }
+
+ @java.lang.Override
+ public void setId(String s) {
+ // unused
+ }
+
+ @java.lang.Override
+ public ArtifactRepositoryPolicy getSnapshots() {
+ return null;
+ }
+
+ @java.lang.Override
+ public void setSnapshotUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
+ // unused
+ }
+
+ @java.lang.Override
+ public ArtifactRepositoryPolicy getReleases() {
+ return null;
+ }
+
+ @java.lang.Override
+ public void setReleaseUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
+ // unused
+ }
+
+ @java.lang.Override
+ public void setLayout(ArtifactRepositoryLayout artifactRepositoryLayout) {
+ // unused
+ }
+
+ @java.lang.Override
+ public String getKey() {
+ return null;
+ }
+
+ @java.lang.Override
+ public boolean isUniqueVersion() {
+ return false;
+ }
+
+ @java.lang.Override
+ public boolean isBlacklisted() {
+ return false;
+ }
+
+ @java.lang.Override
+ public void setBlacklisted(boolean b) {
+ // unused
+ }
+
+ @java.lang.Override
+ public Artifact find(Artifact artifact) {
+ return null;
+ }
+
+ @java.lang.Override
+ public List<String> findVersions(Artifact artifact) {
+ return Collections.emptyList();
+ }
+
+ @java.lang.Override
+ public boolean isProjectAware() {
+ return false;
+ }
+
+ @java.lang.Override
+ public void setAuthentication(Authentication authentication) {
+ // unused
+ }
+
+ @java.lang.Override
+ public Authentication getAuthentication() {
+ return null;
+ }
+
+ @java.lang.Override
+ public void setProxy(Proxy proxy) {
+ // unused
+ }
+
+ @java.lang.Override
+ public Proxy getProxy() {
+ return null;
+ }
+ }
+}