You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by sk...@apache.org on 2020/03/24 17:28:19 UTC
[netbeans] branch master updated: get better list for
mavencoordinates
This is an automated email from the ASF dual-hosted git repository.
skygo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new f350c37 get better list for mavencoordinates
new e0a7250 Merge pull request #2043 from ebarboni/externalupdate
f350c37 is described below
commit f350c3739218b223fe6ce6b178e05c020b30cbdd
Author: Eric Barboni <sk...@apache.org>
AuthorDate: Sun Mar 22 21:43:39 2020 +0100
get better list for mavencoordinates
---
java/maven.embedder/build.xml | 16 +-
java/maven.embedder/external/binariesembedded-list | 60 +++++
.../nbbuild/extlibs/CheckEmbeddedBinaries.java | 254 +++++++++++++++++++++
.../netbeans/nbbuild/extlibs/DownloadBinaries.java | 18 --
nbbuild/build.xml | 24 +-
5 files changed, 351 insertions(+), 21 deletions(-)
diff --git a/java/maven.embedder/build.xml b/java/maven.embedder/build.xml
index 2b8e15b..81572d5 100644
--- a/java/maven.embedder/build.xml
+++ b/java/maven.embedder/build.xml
@@ -27,5 +27,19 @@
<mapper type="glob" from="${bundled.maven}/*" to="*"/>
</unzip>
</target>
-
+ <!-- To check sha1 binaries that are in maven zip (better maven artefacts generation
+ Call manually on maven upgrade
+ -->
+ <target name="checkhash" depends="init">
+ <taskdef name="checkjarssha1" classname="org.netbeans.nbbuild.extlibs.CheckEmbeddedBinaries" classpath="${nbantext.jar}"/>
+ <mkdir dir="tmptesting"/>
+ <unzip src="external/${bundled.maven}-bin.zip" dest="tmptesting">
+ <patternset>
+ <include name="**/*.jar"/>
+ </patternset>
+ <mapper type="flatten"/>
+ </unzip>
+ <checkjarssha1 dir="tmptesting" shalist="external/binariesembedded-list" />
+ <delete dir="tmptesting"/>
+ </target>
</project>
diff --git a/java/maven.embedder/external/binariesembedded-list b/java/maven.embedder/external/binariesembedded-list
new file mode 100644
index 0000000..943fd47
--- /dev/null
+++ b/java/maven.embedder/external/binariesembedded-list
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+44C453F60909DFC223552ACE63E05C694215156B;javax.enterprise:cdi-api:1.0
+C51C00206BB913CD8612B24ABD9FA98AE89719B1;commons-cli:commons-cli:1.4
+2852E6E05FBB95076FC091F6D1780F1F8FE35E0F;commons-io:commons-io:2.5
+6505A72A097D9270F7A9E7BF42C4238283247755;org.apache.commons:commons-lang3:3.8.1
+BDAAB946CA5AD20253502D873BA0C3313D141036;com.google.guava:guava:25.1-android
+41E5AB52EC65E60B6C0CED947BECF7BA96402645;com.google.inject:guice:4.2.1:no_aop
+E90CAA31C9B8D748359450D7487F76B05549AE65;org.fusesource.jansi:jansi:1.17.1
+6975DA39A7040257BD51D21A231B76C915872D38;javax.inject:javax.inject:1
+D6DCA1EC36F28ECF286241DEADC790C2D4C8EAA0;org.slf4j:jcl-over-slf4j:1.7.29
+55819A28FC834C2F2BCF4DCDB278524DC3CF088F;org.jsoup:jsoup:1.12.1
+5025422767732A1AB45D93ABFEA846513D742DCF;javax.annotation:jsr250-api:1.0
+F8FF8032903882376E8D000C51E3E16D20FC7DF7;org.apache.maven:maven-artifact:3.6.3
+E9A37AF390009A525D8FAA6B18BD682123F85F9E;org.apache.maven:maven-builder-support:3.6.3
+2A8242398EFBFD533FFE36147864C34A326666DA;org.apache.maven:maven-compat:3.6.3
+ECA800AA73E750EC9A880EB224F0BB68F5B7873B;org.apache.maven:maven-core:3.6.3
+CC8DF86676BCB37F68D2CEAF05DF8EF6FB11D873;org.apache.maven:maven-embedder:3.6.3
+61C7848DCE2FBF7F7AB0FDC8E8A7CC9DA5DD7827;org.apache.maven:maven-model:3.6.3
+4EF1D56F53D3E0A9003B7CC82C89AF9878321E82;org.apache.maven:maven-model-builder:3.6.3
+063FE5967B9C4C1B6FA6004BE76E1C939E8BD1D6;org.apache.maven:maven-plugin-api:3.6.3
+14D28071C85E76B656C46C465DB91D394D6F48F0;org.apache.maven:maven-repository-metadata:3.6.3
+CEEE6B7EA1BC252AFA585FA32F76C2CDA206BDCD;org.apache.maven.resolver:maven-resolver-api:1.4.1
+C213352D609D576641AED35172157D46CD2003D4;org.apache.maven.resolver:maven-resolver-connector-basic:1.4.1
+1658CFA27978C5949C3A92086514A22CA85394E4;org.apache.maven.resolver:maven-resolver-impl:1.4.1
+115240B65C1D0E9745CB2012B977AFC3D1795F94;org.apache.maven:maven-resolver-provider:3.6.3
+905A024FF050E804A8A4DF53EEEE63CC7D153438;org.apache.maven.resolver:maven-resolver-spi:1.4.1
+F3268F4EE92227AB98B7BE0AC5122DAA2ED97E35;org.apache.maven.resolver:maven-resolver-transport-wagon:1.4.1
+3F6D4F4BC3E24B46A776B47CCFEAED9D2ED01549;org.apache.maven.resolver:maven-resolver-util:1.4.1
+BBF4E06DCDB0BB33D1546C080DF5C8D92B535D30;org.apache.maven:maven-settings:3.6.3
+756D46810B8CC7B2B98585CCC787854CDFDE7FD9;org.apache.maven:maven-settings-builder:3.6.3
+08DD4DFB1D2D8B6969F6462790F82670BCD35CE2;org.apache.maven.shared:maven-shared-utils:3.2.1
+DE3B6CF15C2974250AE88820AB6C210FD8F38ED3;org.apache.maven:maven-slf4j-provider:3.6.3
+FC3BE144183F54DC6F5C55E34462C1C2D89D7D96;org.eclipse.sisu:org.eclipse.sisu.inject:0.3.4
+F1335A3B7BC3FD23F67DA88BD60CD5D4C3304EF3;org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.4
+51460409B6CDC2B828540C19C05691F89141EDC2;org.sonatype.plexus:plexus-cipher:1.7
+8587E80FCB38E70B70FAE8D5914B6376BFAD6259;org.codehaus.plexus:plexus-classworlds:2.6.0
+2F2147A6CC6A119A1B51A96F31D45C557F6244B9;org.codehaus.plexus:plexus-component-annotations:2.1.0
+3B37B3335E6A97E11E690BBDC22ADE1A5DEB74D6;org.codehaus.plexus:plexus-interpolation:1.25
+43FDE524E9B94C883727A9FDDB8669181B890EA7;org.sonatype.plexus:plexus-sec-dispatcher:1.4
+13B015768E0D04849D2794E4C47EB02D01A0DE32;org.codehaus.plexus:plexus-utils:3.2.1
+E56BF4473A4C6B71C7DD397A833DCE86D1993D9D;org.slf4j:slf4j-api:1.7.29
+9F4E71EFC815C087E32C2C80341C1ED5D9CBEDD6;org.apache.maven.wagon:wagon-file:3.3.4
+8152D42E1C416AF9B4B146FB1AA082BBCC55C557;org.apache.maven.wagon:wagon-http:3.3.4:shaded
+7A99FDAA534AA8C01F01A447FE1C7AF5CFC7B0D5;org.apache.maven.wagon:wagon-provider-api:3.3.4
\ No newline at end of file
diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/CheckEmbeddedBinaries.java b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/CheckEmbeddedBinaries.java
new file mode 100644
index 0000000..94db7ba
--- /dev/null
+++ b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/CheckEmbeddedBinaries.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.nbbuild.extlibs;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.Task;
+
+/**
+ * Task to check sha1 from named files (generally binaries such as ZIPs)
+ * from a repository.
+ */
+public class CheckEmbeddedBinaries extends Task {
+
+ private File dir;
+
+ /**
+ * Location of unzippped jar folder to be tested.
+ */
+ public void setDir(File dir) {
+ this.dir = dir;
+ }
+
+ private File shalist;
+
+ /**
+ * List of chechcksum and coordinate
+ * @param shaList
+ */
+ public void setShaList(File shaList) {
+ this.shalist = shaList;
+ }
+
+ @Override
+ public void execute() throws BuildException {
+ boolean success = true;
+
+ File manifest = shalist;
+ Map<String,String> shamap = new HashMap<>();
+ log("Scanning: " + manifest, Project.MSG_VERBOSE);
+ try {
+ try (InputStream is = new FileInputStream(manifest)) {
+ BufferedReader r = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+ String line;
+ while ((line = r.readLine()) != null) {
+ if (line.startsWith("#")) {
+ continue;
+ }
+ if (line.trim().length() == 0) {
+ continue;
+ }
+ String[] hashAndFile = line.split(";", 2);
+ if (hashAndFile.length < 2) {
+ throw new BuildException("Bad line '" + line + "' in " + manifest, getLocation());
+ }
+
+ if (MavenCoordinate.isMavenFile(hashAndFile[1])) {
+ MavenCoordinate mc = MavenCoordinate.fromGradleFormat(hashAndFile[1]);
+ shamap.put(hashAndFile[0], hashAndFile[1]);
+ } else {
+ throw new BuildException("Invalid manifest entry should be Maven coordinate", getLocation());
+ }
+ }
+ }
+ } catch (IOException x) {
+ throw new BuildException("Could not open " + manifest + ": " + x, x, getLocation());
+
+ }
+ try {
+ StringBuilder errorList = new StringBuilder();
+ Files.list(dir.toPath())
+ .forEach((t) -> {
+ String sha1 = hash(t.toFile());
+ if (!shamap.containsKey(sha1)) {
+ errorList.append("No sha1 (expected ").append(sha1).append(" for file: ").append(t).append("\n");
+ }
+ });
+ if (errorList.toString().length()>0) {
+ log(""+errorList.toString());
+ throw new BuildException("Missing Sha1 file", getLocation());
+ }
+ } catch (IOException ex) {
+ throw new BuildException("Invalid manifest entry should be Maven coordinate", getLocation());
+ }
+ if (!success) {
+ throw new BuildException("Failed to download binaries - see log message for the detailed reasons.", getLocation());
+ }
+ }
+
+ private String hash(File f) {
+ try {
+ try (FileInputStream is = new FileInputStream(f)) {
+ return hash(is);
+ }
+ } catch (IOException x) {
+ throw new BuildException("Could not get hash for " + f + ": " + x, x, getLocation());
+ }
+ }
+
+ private String hash(InputStream is) throws IOException {
+ MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance("SHA-1");
+ } catch (NoSuchAlgorithmException x) {
+ throw new BuildException(x, getLocation());
+ }
+ byte[] buf = new byte[4096];
+ int r;
+ while ((r = is.read(buf)) != -1) {
+ digest.update(buf, 0, r);
+ }
+ return String.format("%040X", new BigInteger(1, digest.digest()));
+ }
+
+ static class MavenCoordinate {
+
+ private final String groupId;
+ private final String artifactId;
+ private final String version;
+ private final String extension;
+ private final String classifier;
+
+ private MavenCoordinate(String groupId, String artifactId, String version, String extension, String classifier) {
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.version = version;
+ this.extension = extension;
+ this.classifier = classifier;
+ }
+
+ public boolean hasClassifier() {
+ return (!classifier.isEmpty());
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public String getArtifactId() {
+ return artifactId;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getExtension() {
+ return extension;
+ }
+
+ public String getClassifier() {
+ return classifier;
+ }
+
+ /**
+ * @return filename of the artifact by maven convention:
+ * {@code artifact-version[-classifier].extension}
+ */
+ public String toArtifactFilename() {
+ return String.format("%s-%s%s.%s",
+ getArtifactId(),
+ getVersion(),
+ hasClassifier() ? ("-" + getClassifier()) : "",
+ getExtension()
+ );
+ }
+
+ /**
+ * @return The repository path for an artifact by maven convention:
+ * {@code group/artifact/version/artifact-version[-classifier].extension}.
+ * In the group part all dots are replaced by a slash.
+ */
+ public String toMavenPath() {
+ return String.format("%s/%s/%s/%s",
+ getGroupId().replace(".", "/"),
+ getArtifactId(),
+ getVersion(),
+ toArtifactFilename()
+ );
+ }
+
+ public static boolean isMavenFile(String gradleFormat) {
+ return gradleFormat.split(":").length > 2;
+ }
+
+ /**
+ * The maven coordinate is supplied in the form:
+ *
+ * <p>
+ * {@code group:name:version:classifier@extension}</p>
+ *
+ * <p>
+ * For the DownloadBinaries task the parts group, name and version are
+ * requiered. classifier and extension are optional. The extension has a
+ * default value of "jar".
+ *
+ * @param gradleFormat artifact coordinated to be parse as a
+ * MavenCoordinate
+ * @return
+ * @throws IllegalArgumentException if provided string fails to parse
+ */
+ public static MavenCoordinate fromGradleFormat(String gradleFormat) {
+ if (!isMavenFile(gradleFormat)) {
+ throw new IllegalArgumentException("Supplied string is not in gradle dependency format: " + gradleFormat);
+ }
+ String[] coordinateExtension = gradleFormat.split("@", 2);
+ String extension;
+ String coordinate = coordinateExtension[0];
+ if (coordinateExtension.length > 1
+ && (!coordinateExtension[1].trim().isEmpty())) {
+ extension = coordinateExtension[1];
+ } else {
+ extension = "jar";
+ }
+ String[] coordinates = coordinate.split(":");
+ String group = coordinates[0];
+ String artifact = coordinates[1];
+ String version = coordinates[2];
+ String classifier = "";
+ if (coordinates.length > 3) {
+ classifier = coordinates[3].trim();
+ }
+ return new MavenCoordinate(group, artifact, version, extension, classifier);
+ }
+ }
+}
diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DownloadBinaries.java b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DownloadBinaries.java
index 7fae888..355b90e 100644
--- a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DownloadBinaries.java
+++ b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DownloadBinaries.java
@@ -95,12 +95,6 @@ public class DownloadBinaries extends Task {
this.repos = repos;
}
- private File report = null;
-
- public void setReportToFile(File file) {
- this.report = file;
- }
-
private final List<FileSet> manifests = new ArrayList<>();
/**
* Add one or more manifests of files to download.
@@ -141,14 +135,6 @@ public class DownloadBinaries extends Task {
@Override
public void execute() throws BuildException {
boolean success = true;
- if (report != null) {
- try {
- Files.deleteIfExists(report.toPath());
- Files.createFile(report.toPath());
- } catch (IOException x) {
- throw new BuildException("Cannot create report file at : " + report + x, getLocation());
- }
- }
for (FileSet fs : manifests) {
DirectoryScanner scanner = fs.getDirectoryScanner(getProject());
File basedir = scanner.getBasedir();
@@ -177,10 +163,6 @@ public class DownloadBinaries extends Task {
} else {
success &= fillInFile(hashAndFile[0], hashAndFile[1], manifest, () -> legacyDownload(hashAndFile[0] + "-" + hashAndFile[1]));
}
- if (report != null) {
- Files.write(report.toPath(), (hashAndFile[0] + ";" + hashAndFile[1] + ";" + include + "\n").getBytes("UTF-8"),StandardOpenOption.APPEND);
- }
-
}
}
} catch (IOException x) {
diff --git a/nbbuild/build.xml b/nbbuild/build.xml
index 99f95da..45bc1f3 100644
--- a/nbbuild/build.xml
+++ b/nbbuild/build.xml
@@ -198,14 +198,34 @@ metabuild.hash=${metabuild.hash}</echo>
<target name="download-all-extbins" unless="ext.binaries.downloaded" depends="bootstrap">
<echo>Downloading external binaries (*/external/ directories)...</echo>
<!-- optionnal reporttofile used to speed resolving artefacts resolution for maven artefacts -->
- <downloadbinaries cache="${binaries.cache}" server="${binaries.server}" reporttofile="${nb_all}/nbbuild/build/external.info" >
+ <downloadbinaries cache="${binaries.cache}" server="${binaries.server}" >
<manifest dir="${nb_all}">
<include name="**/external/binaries-list"/>
</manifest>
</downloadbinaries>
<property name="ext.binaries.downloaded" value="true"/>
</target>
-
+ <!-- create list of all maven coordinate -->
+ <target name="getallmavencoordinates">
+ <concat destfile="${nb_all}/nbbuild/build/external.info">
+ <fileset dir="${nb_all}" includes="**/external/binaries-list" />
+ <fileset dir="${nb_all}" includes="**/external/binariesembedded-list" />
+ <filterchain>
+ <!-- remove comment inline not inline -->
+ <tokenfilter>
+ <replaceregex pattern="#(.*)$" replace="" flags="gim" />
+ </tokenfilter>
+ <!-- : is sparator for maven coordinate -->
+ <linecontains>
+ <contains value=":"/>
+ </linecontains>
+ <!-- needed as separator -->
+ <tokenfilter>
+ <replacestring from=" " to=";"/>
+ </tokenfilter>
+ </filterchain>
+ </concat>
+ </target>
<target name="download-selected-extbins" unless="ext.binaries.downloaded" depends="init-module-list">
<echo>Downloading external binaries (*/external/ directories) for cluster.config=${cluster.config}...</echo>
<pathconvert property="modules.binaries-list" pathsep=",">
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@netbeans.apache.org
For additional commands, e-mail: commits-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists