You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2018/09/13 12:49:18 UTC
[24/50] [abbrv] ignite git commit: IGNITE-9388: mesos IgniteProvider
tries to access obolete ignite.run or download from slow archive. - Fixes
#4639.
IGNITE-9388: mesos IgniteProvider tries to access obolete ignite.run or download from slow archive. - Fixes #4639.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/09a823cf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/09a823cf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/09a823cf
Branch: refs/heads/ignite-9349
Commit: 09a823cf6c435372f75d53f5ddb9d15ce0c3e55b
Parents: 69ac976
Author: shroman <rs...@yahoo.com>
Authored: Wed Sep 12 10:10:56 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Wed Sep 12 10:10:56 2018 +0900
----------------------------------------------------------------------
modules/mesos/pom.xml | 36 ++++--
.../ignite/mesos/resource/IgniteProvider.java | 120 ++++++++++++++++---
2 files changed, 127 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/09a823cf/modules/mesos/pom.xml
----------------------------------------------------------------------
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index a8c7d8b..a5c711a 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -36,8 +36,9 @@
<properties>
<mesos.version>1.5.0</mesos.version>
- <ignite.latest.url>http://ignite.run/download_ignite.php</ignite.latest.url>
- <ignite.direct.url>https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.direct.url>
+ <ignite.version.url>https://ignite.apache.org/latest</ignite.version.url>
+ <ignite.path>/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.path>
+ <apache.mirror.url>https://www.apache.org/dyn/closer.cgi?as_json=1</apache.mirror.url>
</properties>
<dependencies>
@@ -54,6 +55,12 @@
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
@@ -76,23 +83,30 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <echo message="Update download url in mesos module." />
- <echo message="Direct link ${ignite.direct.url}." />
- <echo message="Latest link ${ignite.latest.url}." />
+ <echo message="Update download url in mesos module."/>
+ <echo message="Latest version ${ignite.version.url}."/>
+
+ <replaceregexp byline="true" encoding="UTF-8">
+ <regexp pattern="(.*IGNITE_LATEST_VERSION_URL = ")(.*)(".*)"/>
+ <substitution expression="\1${ignite.version.url}\3"/>
+ <fileset dir="${basedir}/">
+ <include name="**/IgniteProvider.java"/>
+ </fileset>
+ </replaceregexp>
<replaceregexp byline="true" encoding="UTF-8">
- <regexp pattern="(.*DOWNLOAD_LINK = ")(.*)(".*)" />
- <substitution expression="\1${ignite.latest.url}\3" />
+ <regexp pattern="(.*APACHE_MIRROR_URL = ")(.*)(".*)"/>
+ <substitution expression="\1${apache.mirror.url}\3"/>
<fileset dir="${basedir}/">
- <include name="**/IgniteProvider.java" />
+ <include name="**/IgniteProvider.java"/>
</fileset>
</replaceregexp>
<replaceregexp byline="true" encoding="UTF-8">
- <regexp pattern="(.*DOWNLOAD_URL_PATTERN = ")(.*)(".*)" />
- <substitution expression="\1${ignite.direct.url}\3" />
+ <regexp pattern="(.*IGNITE_PATH = ")(.*)(".*)"/>
+ <substitution expression="\1${ignite.path}\3"/>
<fileset dir="${basedir}/">
- <include name="**/IgniteProvider.java" />
+ <include name="**/IgniteProvider.java"/>
</fileset>
</replaceregexp>
</target>
http://git-wip-us.apache.org/repos/asf/ignite/blob/09a823cf/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
index 3d794db..5735051 100644
--- a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java
@@ -17,9 +17,13 @@
package org.apache.ignite.mesos.resource;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.Channels;
@@ -27,19 +31,38 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import org.apache.ignite.mesos.ClusterProperties;
+import static org.apache.ignite.mesos.ClusterProperties.IGNITE_VERSION;
+
/**
* Class downloads and stores Ignite.
*/
public class IgniteProvider {
- // This constants are set by maven-ant-plugin.
- /** */
- private static final String DOWNLOAD_LINK = "http://ignite.run/download_ignite.php";
+ /** Logger. */
+ private static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName());
+ // This constants are set by maven-ant-plugin.
/** */
private static final String DOWNLOAD_URL_PATTERN = "https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip";
+ /** URL for request Ignite latest version. */
+ private final static String IGNITE_LATEST_VERSION_URL = "";
+
+ /** Mirrors. */
+ private static final String APACHE_MIRROR_URL = "";
+
+ /** Ignite on Apache URL path. */
+ private static final String IGNITE_PATH = "";
+
+ /** Version pattern. */
+ private static final Pattern VERSION_PATTERN = Pattern.compile("(?<=version=).*\\S+");
+
/** */
private String downloadFolder;
@@ -52,8 +75,8 @@ public class IgniteProvider {
/**
* @param ver Ignite version.
- * @throws IOException If downloading failed.
* @return Path to latest ignite.
+ * @throws IOException If downloading failed.
*/
public String getIgnite(String ver) throws IOException {
return downloadIgnite(ver);
@@ -61,31 +84,90 @@ public class IgniteProvider {
/**
* @param ver Ignite version which will be downloaded. If {@code null} will download the latest ignite version.
- * @throws IOException If downloading failed.
* @return Ignite archive.
+ * @throws IOException If downloading failed.
*/
public String downloadIgnite(String ver) throws IOException {
assert ver != null;
URL url;
+ // get the latest version.
if (ver.equals(ClusterProperties.DEFAULT_IGNITE_VERSION)) {
- URL updateUrl = new URL(DOWNLOAD_LINK);
+ try {
+ ver = findLatestVersion();
- HttpURLConnection conn = (HttpURLConnection)updateUrl.openConnection();
+ // and try to retrieve from a mirror.
+ url = new URL(String.format(findMirror() + IGNITE_PATH, ver, ver));
+ }
+ catch (Exception e) {
+ // fallback to archive.
+ url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver, ver));
+ }
+ }
+ else {
+ // or from archive.
+ url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver, ver));
+ }
- int code = conn.getResponseCode();
+ return downloadIgnite(url);
+ }
- if (code == 200)
- url = conn.getURL();
- else
- throw new RuntimeException("Failed to download ignite distributive. Maybe set incorrect version? " +
- "[resCode:" + code + ", ver: " + ver + "]");
- }
+ /**
+ * Attempts to retrieve the preferred mirror.
+ *
+ * @return Mirror url.
+ * @throws IOException If failed.
+ */
+ private String findMirror() throws IOException {
+ String response = getHttpContents(new URL(APACHE_MIRROR_URL));
+
+ if (response == null)
+ throw new RuntimeException("Failed to retrieve mirrors");
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode mirrorUrl = mapper.readTree(response).get("preferred");
+
+ if (mirrorUrl == null)
+ throw new RuntimeException("Failed to find the preferred mirror");
+
+ return mirrorUrl.asText();
+ }
+
+ /**
+ * Attempts to obtain the latest version.
+ *
+ * @return Latest version.
+ * @throws IOException If failed.
+ */
+ private String findLatestVersion() throws IOException {
+ String response = getHttpContents(new URL(IGNITE_LATEST_VERSION_URL));
+
+ if (response == null)
+ throw new RuntimeException("Failed to identify the latest version. Specify it with " + IGNITE_VERSION);
+
+ Matcher m = VERSION_PATTERN.matcher(response);
+ if (m.find())
+ return m.group();
else
- url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver.replace("-incubating", ""), ver));
+ throw new RuntimeException("Failed to retrieve the latest version. Specify it with " + IGNITE_VERSION);
+ }
- return downloadIgnite(url);
+ /**
+ * @param url Url.
+ * @return Contents.
+ * @throws IOException If failed.
+ */
+ private String getHttpContents(URL url) throws IOException {
+ HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+
+ int code = conn.getResponseCode();
+
+ if (code != 200)
+ throw null;
+
+ BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
+ return rd.lines().collect(Collectors.joining());
}
/**
@@ -110,7 +192,9 @@ public class IgniteProvider {
if (fileExist(fileName))
return fileName;
- FileOutputStream outFile = new FileOutputStream(downloadFolder + fileName);
+ log.log(Level.INFO, "Downloading from {0}", url.toString());
+
+ FileOutputStream outFile = new FileOutputStream(Paths.get(downloadFolder, fileName).toFile());
outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
@@ -119,7 +203,7 @@ public class IgniteProvider {
return fileName;
}
else
- throw new RuntimeException("Got unexpected response code. Response code: " + code);
+ throw new RuntimeException("Got unexpected response code. Response code: " + code + " from " + url);
}
catch (IOException e) {
throw new RuntimeException("Failed to download Ignite.", e);