You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tomee.apache.org by rz...@apache.org on 2021/03/26 07:54:18 UTC
[tomee-site-generator] branch master updated: DownloadsNG:
Incorporates some code-related feedback received via the mailing list,
reduces code duplication.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tomee-site-generator.git
The following commit(s) were added to refs/heads/master by this push:
new 9fc81ea DownloadsNG: Incorporates some code-related feedback received via the mailing list, reduces code duplication.
9fc81ea is described below
commit 9fc81ea89fcc652b9ad76198189e265d7219683b
Author: Richard Zowalla <ri...@hs-heilbronn.de>
AuthorDate: Fri Mar 26 08:53:55 2021 +0100
DownloadsNG: Incorporates some code-related feedback received via the mailing list, reduces code duplication.
---
.../apache/tomee/website/AbstractDownloadsNG.java | 187 +++++++++++----------
.../java/org/apache/tomee/website/DownloadsNG.java | 35 ++--
2 files changed, 111 insertions(+), 111 deletions(-)
diff --git a/src/main/java/org/apache/tomee/website/AbstractDownloadsNG.java b/src/main/java/org/apache/tomee/website/AbstractDownloadsNG.java
index f117685..a96f8c7 100644
--- a/src/main/java/org/apache/tomee/website/AbstractDownloadsNG.java
+++ b/src/main/java/org/apache/tomee/website/AbstractDownloadsNG.java
@@ -22,6 +22,7 @@ import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
@@ -37,7 +38,6 @@ import static java.util.Optional.ofNullable;
import static java.util.stream.Collectors.toList;
import static lombok.AccessLevel.PROTECTED;
-// regenerate active download page using mirroring system
@RequiredArgsConstructor(access = PROTECTED)
public class AbstractDownloadsNG {
@@ -47,12 +47,12 @@ public class AbstractDownloadsNG {
private static final String BASE_MIRROR_URL = "https://www.apache.org/dyn/closer.cgi/tomee/";
public static void generate(String baseUrl) throws MalformedURLException {
- List<AbstractDownloadsNG.Download> downloads = getDownloadables(baseUrl);
+ final List<AbstractDownloadsNG.Download> downloads = getDownloadables(baseUrl);
- AtomicReference<String> version1 = new AtomicReference<>();
- AtomicReference<String> version7 = new AtomicReference<>();
- AtomicReference<String> version8 = new AtomicReference<>();
- AtomicReference<String> version9 = new AtomicReference<>();
+ final AtomicReference<String> version1 = new AtomicReference<>();
+ final AtomicReference<String> version7 = new AtomicReference<>();
+ final AtomicReference<String> version8 = new AtomicReference<>();
+ final AtomicReference<String> version9 = new AtomicReference<>();
List<AbstractDownloadsNG.Download> filled = downloads.stream().parallel()
.map(AbstractDownloadsNG::fillDownloadable)
@@ -106,9 +106,9 @@ public class AbstractDownloadsNG {
}
}
- private static void printRow(AbstractDownloadsNG.Download d) {
+ private static void printRow(final AbstractDownloadsNG.Download d) {
- String hash = d.sha512.isEmpty() ? (d.sha256.isEmpty() ? (d.sha1.isEmpty() ? "" : d.sha1 + "[icon:download[] SHA1] ") : d.sha256 + "[icon:download[] SHA256] ") : d.sha512 + "[icon:download[] SHA512] ";
+ final String hash = d.sha512.isEmpty() ? (d.sha256.isEmpty() ? (d.sha1.isEmpty() ? "" : d.sha1 + "[icon:download[] SHA1] ") : d.sha256 + "[icon:download[] SHA256] ") : d.sha512 + "[icon:download[] SHA512] ";
System.out.println("" +
"|" + d.mirrorUrl + "[icon:download[] " + d.name.replace("Apache ", "") + " " + d.format.toUpperCase() + "] " +
@@ -118,85 +118,96 @@ public class AbstractDownloadsNG {
"|" + d.asc + "[icon:download[] PGP] " + hash);
}
- private static List<AbstractDownloadsNG.Download> getDownloadables(String base) throws MalformedURLException {
- QuickDistLinkParser quickDistLinkParser = new QuickDistLinkParser();
+ private static List<AbstractDownloadsNG.Download> getDownloadables(final String base) throws MalformedURLException {
+ final QuickDistLinkParser quickDistLinkParser = new QuickDistLinkParser();
- Results results = quickDistLinkParser.fetch(new URL(base));
+ final Results results = quickDistLinkParser.fetch(new URL(base));
- List<AbstractDownloadsNG.Download> downloads = new ArrayList<>();
+ final List<AbstractDownloadsNG.Download> downloads = new ArrayList<>();
//parse the links, generate download set
for (Link distLink : results.links) {
- String version = distLink.getUri().replaceAll("tomee-", "").replaceAll("/", "");
+ final String version = getVersionFromURI(distLink.getUri());
- Results distResults = quickDistLinkParser.fetch(new URL(base + distLink.getUri()));
+ final Results distResults = quickDistLinkParser.fetch(new URL(base + distLink.getUri()));
- List<Link> links = distResults.links
+ final List<Link> links = distResults.links
.stream()
- .filter(l -> l.getUri().endsWith(".tar.gz")
- || l.getUri().endsWith(".zip")
- || l.getUri().endsWith(".war"))
+ .filter(l -> l.getUri().endsWith("tar.gz")
+ || l.getUri().endsWith("zip")
+ || l.getUri().endsWith("war"))
.collect(Collectors.toList());
- Set<String> available = distResults.links
+ final Set<String> available = distResults.links
.stream()
.map(l -> distResults.baseUrl + l.getUri())
.collect(Collectors.toSet());
for (Link link : links) {
- String uri = link.getUri();
- String baseName = "";
- String extension = "";
-
- if (uri.contains(".tar.gz")) {
- baseName = uri.substring(0, uri.lastIndexOf(".tar.gz"));
- extension = uri.substring(uri.lastIndexOf("tar.gz"));
- } else if (uri.contains(".zip")) {
- baseName = uri.substring(0, uri.lastIndexOf(".zip"));
- extension = uri.substring(uri.lastIndexOf("zip"));
- } else if (uri.contains(".war")) {
- baseName = uri.substring(0, uri.lastIndexOf(".war"));
- extension = uri.substring(uri.lastIndexOf("war"));
- } else {
- System.err.println("This should not happen. New extensions?");
- }
-
- String url = distResults.baseUrl + uri;
- String ascURL = url + ".asc";
- String sha1URL = url + ".sha1";
- String sha256URL = url + ".sha256";
- String sha512URL = url + ".sha512";
-
- if (!available.contains(ascURL)) {
- System.err.println("Release " + ascURL + " does not have a ASC signature. This shouldn't happen!");
- throw new RuntimeException("Release does not have a ASC signature. This shouldn't happen!");
- }
+ final String url = distResults.baseUrl + link.getUri();
+ final String baseName = getBaseNameFromURI(link.getUri());
+ final String extension = getExtensionFromURI(link.getUri());
+ final String ascURL = checkURLAvailable(url + ".asc", available, true);
+ final String sha1URL = checkURLAvailable(url + ".sha1", available, false);
+ final String sha256URL = checkURLAvailable(url + ".sha256", available, false);
+ final String sha512URL = checkURLAvailable(url + ".sha512", available, false);
- if (!available.contains(sha1URL)) {
- sha1URL = "";
+ if (sha1URL.isEmpty() && sha256URL.isEmpty() && sha512URL.isEmpty()) {
+ throw new RuntimeException("Release " + baseName + " does not have a valid hash. This shouldn't happen!");
}
- if (!available.contains(sha256URL)) {
- sha256URL = "";
- }
+ downloads.add(toDownload(baseName, version, extension, url, sha1URL, sha256URL, sha512URL, ascURL));
+ }
+ }
+ return downloads;
+ }
- if (!available.contains(sha512URL)) {
- sha512URL = "";
- }
+ /**
+ * Checks, if we encountered the given url in the parsing results of the web directory content.
+ *
+ * @param possibleURL the url to check
+ * @param availableURLs a {@link Set} containing the identified links of the web directory content.
+ * @param mandatory if {@code true}, an exception is thrown if the given URL is not contained in the given set.
+ * @return {@code ""}, if the given URL is not contained and would be a dead link.
+ */
+ private static String checkURLAvailable(final String possibleURL, final Set<String> availableURLs, final boolean mandatory) {
+ if (!availableURLs.contains(possibleURL)) {
+ if (mandatory) {
+ throw new RuntimeException("Mandatory file " + possibleURL + " is not available. This shouldn't happen!");
+ }
+ return "";
+ }
+ return possibleURL;
+ }
- if (sha1URL.isEmpty() && sha256URL.isEmpty() && sha512URL.isEmpty()) {
- System.err.println("Release " + baseName + " does not have a valid hash. This shouldn't happen!");
- throw new RuntimeException("Release does not have a valid hash. This shouldn't happen!");
- }
+ private static String getVersionFromURI(final String uri) {
+ return uri.replaceAll("tomee-", "").replaceAll("/", "");
+ }
- AbstractDownloadsNG.Download dl = toDownload(baseName, version, extension, url, sha1URL, sha256URL, sha512URL, ascURL);
+ private static String getExtensionFromURI(final String uri) {
+ if (uri.endsWith("tar.gz")) {
+ return uri.substring(uri.lastIndexOf("tar.gz"));
+ } else if (uri.endsWith("zip")) {
+ return uri.substring(uri.lastIndexOf("zip"));
+ } else if (uri.endsWith("war")) {
+ return uri.substring(uri.lastIndexOf("war"));
+ } else {
+ throw new RuntimeException("Release links have an unknown extension type. New release extensions???");
+ }
+ }
- downloads.add(dl);
- }
+ private static String getBaseNameFromURI(final String uri) {
+ if (uri.endsWith("tar.gz")) {
+ return uri.substring(0, uri.lastIndexOf("tar.gz") - 1);
+ } else if (uri.endsWith("zip")) {
+ return uri.substring(0, uri.lastIndexOf("zip") - 1);
+ } else if (uri.endsWith("war")) {
+ return uri.substring(0, uri.lastIndexOf("war") - 1);
+ } else {
+ throw new RuntimeException("Release links have an unknown extension type. New release extensions???");
}
- return downloads;
}
private static AbstractDownloadsNG.Download fillDownloadable(final AbstractDownloadsNG.Download download) {
@@ -244,7 +255,10 @@ public class AbstractDownloadsNG {
asc);
}
- private static void checkMaxVersion(AtomicReference<String> version1, AtomicReference<String> version7, AtomicReference<String> version8, AtomicReference<String> version9, AbstractDownloadsNG.Download o1, int versionComp) {
+ private static void checkMaxVersion(final AtomicReference<String> version1,
+ final AtomicReference<String> version7,
+ final AtomicReference<String> version8,
+ final AtomicReference<String> version9, final AbstractDownloadsNG.Download o1, final int versionComp) {
if (versionComp > 0) {
if (o1.version.startsWith("1.")) {
version1.set(o1.version);
@@ -258,14 +272,12 @@ public class AbstractDownloadsNG {
}
}
-
// distinct by property in a Java stream
- private static <T> Predicate<T> distinctBy(Function<? super T, ?> keyExtractor) {
+ private static <T> Predicate<T> distinctBy(final Function<? super T, ?> keyExtractor) {
final Set<Object> seen = new HashSet<>();
return t -> seen.add(keyExtractor.apply(t));
}
-
@Data
public static class Download {
private final String name;
@@ -277,35 +289,35 @@ public class AbstractDownloadsNG {
private final String sha512;
private final String asc;
private String mirrorUrl;
- private String date; /* "Wed, 05 Aug 2020 16:26:51 GMT" */
- private long size;
+ private String date = "Wed, 05 Aug 2020 16:26:51 GMT";
+ private long size = 10;
}
private static class QuickDistLinkParser {
- public Results fetch(URL url) {
- Results results = new Results();
+ public Results fetch(final URL url) {
- try (InputStream input = url.openStream()) {
+ try (final InputStream input = url.openStream()) {
- LinkContentHandler linkHandler = new LinkContentHandler();
- Metadata metadata = new Metadata();
- HtmlParser parser = new HtmlParser();
+ final LinkContentHandler linkHandler = new LinkContentHandler();
+ final Metadata metadata = new Metadata();
+ final HtmlParser parser = new HtmlParser();
- ContentHandler textHandler = new BodyContentHandler(-1);
+ final ContentHandler textHandler = new BodyContentHandler(-1);
parser.parse(input,
new TeeContentHandler(linkHandler, textHandler),
metadata,
new ParseContext());
+ //TODO Possible Improvement: Filters (see below) as CLI parameters
- results.baseUrl = url.toString();
// keep only <a> tags with non-empty href + avoid duplicates + only links containing "tomee" or "openejb", filtering some other artifacts on the dist server
- results.links = linkHandler.getLinks().stream()
+ final List<Link> downloadLinks = linkHandler.getLinks().stream()
.filter(l -> l.isAnchor() &&
- (l.getUri().contains("tomee-") || l.getUri().contains("openejb-standalone")) &&
!l.getUri().isEmpty() &&
+ !l.getUri().startsWith("#") &&
+ (l.getUri().contains("tomee-") || l.getUri().contains("openejb-standalone")) &&
// release >= 1.7.x contains this -> is not linked on website -> filter it
!l.getUri().contains("arquillian") &&
// release >= 4.7.x contains this -> is not linked on website -> filter it
@@ -322,29 +334,24 @@ public class AbstractDownloadsNG {
!l.getUri().contains("apache-tomee-9.0.0-M2-source-release") &&
// release 9.0.0-M1 has no (real) source release -> filter it
!l.getUri().contains("apache-tomee-9.0.0-M1-source-release") &&
- // no need to have hashes of sigs
+ // no need to have signature hashes
!l.getUri().endsWith("asc.sha1") &&
!l.getUri().endsWith("asc.sha256") &&
- !l.getUri().endsWith("asc.sha512") &&
- !l.getUri().startsWith("#"))
+ !l.getUri().endsWith("asc.sha512"))
.filter(distinctBy(Link::getUri))
.collect(Collectors.toList());
- return results;
+ return new Results(url.toString(), Collections.unmodifiableList(downloadLinks));
} catch (Exception e) {
- e.printStackTrace();
+ throw new RuntimeException("Could not obtain download link. See stacktrace for further information.", e);
}
- return null;
}
}
-
+ @Data
public static class Results {
- public String baseUrl;
- public List<Link> links;
-
- Results() {
- }
+ private final String baseUrl;
+ private final List<Link> links;
@Override
public String toString() {
diff --git a/src/main/java/org/apache/tomee/website/DownloadsNG.java b/src/main/java/org/apache/tomee/website/DownloadsNG.java
index 660b49f..1cbeab8 100644
--- a/src/main/java/org/apache/tomee/website/DownloadsNG.java
+++ b/src/main/java/org/apache/tomee/website/DownloadsNG.java
@@ -11,41 +11,34 @@ import static lombok.AccessLevel.PUBLIC;
@RequiredArgsConstructor(access = PUBLIC)
public class DownloadsNG extends AbstractDownloadsNG {
- public static void main(final String[] args) throws IOException, TikaException, SAXException {
+ public static void main(final String[] args) throws Exception {
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "512");
boolean archive = false;
- if(args.length > 0) {
+ if (args.length > 0) {
archive = Boolean.parseBoolean(args[0]);
}
- if(archive) {
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
+ if (archive) {
+ printSeparator(4);
System.out.println("Please note, that versions prior to 1.5.1 are not available on archive.apache.org! You need to reference them via repo.maven.apache.org.");
+ //TODO Improvement: Remove current release versions before printing the archives
System.out.println("Please remove current versions from the output...");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
- //TODO Possible Improvement: Remove current release versions before out putting them for the archives ;)
+ printSeparator(4);
generate(ARCHIVE_BASE_URL);
- }else {
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
+ } else {
+ printSeparator(4);
System.out.println("Generating current download links targeting the mirroring system.");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
- System.out.println("###################");
+ printSeparator(4);
generate(BASE_URL);
}
+ }
+ private static void printSeparator(int times) {
+ for(int i = 0; i < times; i ++) {
+ System.out.println("###################");
+ }
}
}