You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by il...@apache.org on 2019/06/13 15:18:48 UTC
[ignite] branch master updated: IGNITE-9409 Use current version,
modern URL for Ignite download - Fixes #6599.
This is an automated email from the ASF dual-hosted git repository.
ilyak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new ebb3659 IGNITE-9409 Use current version, modern URL for Ignite download - Fixes #6599.
ebb3659 is described below
commit ebb3659f4346b4d86c1460b764a030dd54d2b394
Author: Ilya Kasnacheev <il...@gmail.com>
AuthorDate: Thu Jun 13 18:18:10 2019 +0300
IGNITE-9409 Use current version, modern URL for Ignite download - Fixes #6599.
Signed-off-by: Ilya Kasnacheev <il...@gmail.com>
---
modules/yarn/pom.xml | 25 +++
.../org/apache/ignite/yarn/IgniteProvider.java | 217 +++++++--------------
.../org/apache/ignite/yarn/IgniteYarnClient.java | 12 +-
modules/yarn/src/main/resources/ignite.properties | 23 +++
4 files changed, 130 insertions(+), 147 deletions(-)
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index 6c95a82..3d3642e 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -100,6 +100,31 @@
<build>
<plugins>
<plugin>
+ <!-- Copy ignite.properties from core module since it is not available as dependency. -->
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>src/main/resources</outputDirectory>
+ <resources>
+ <resource>
+ <directory>../core/src/main/resources</directory>
+ <includes>
+ <include>ignite.properties</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
index d4faa38..311058c 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
@@ -21,33 +21,29 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.Channels;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
+import java.util.Properties;
+import java.util.logging.Logger;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.ignite.yarn.utils.IgniteYarnUtils;
/**
- * Downloads and stores Ignite.
+ * Downloads and stores Ignite release.
*/
public class IgniteProvider {
/** */
- public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php";
+ public static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName());
/** */
- private ClusterProperties props;
-
- /** */
- private String latestVersion = null;
+ public static final String DOWNLOAD_LINK = "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/";
/** */
- private boolean hdfs = false;
+ private ClusterProperties props;
/** */
private FileSystem fs;
@@ -62,93 +58,67 @@ public class IgniteProvider {
}
/**
- * @return Latest ignite version.
+ * @return Path to current Ignite release.
*/
public Path getIgnite() throws Exception {
File folder = checkDownloadFolder();
- if (latestVersion == null) {
- List<String> localFiles = findIgnites(folder);
- List<String> hdfsFiles = findIgnites(fs, props.igniteReleasesDir());
+ Properties verProps = new Properties();
- String localLatestVersion = findLatestVersion(localFiles);
- String hdfsLatestVersion = findLatestVersion(hdfsFiles);
+ try (InputStream is = IgniteProvider.class.getClassLoader().getResourceAsStream("ignite.properties")) {
+ verProps.load(is);
+ }
- if (localLatestVersion != null && hdfsLatestVersion != null) {
- if (VersionComparator.INSTANCE.compare(hdfsLatestVersion, localLatestVersion) >= 0) {
- latestVersion = hdfsLatestVersion;
+ String curVer = verProps.getProperty("ignite.version");
- hdfs = true;
- }
- }
- else if (localLatestVersion != null)
- latestVersion = localLatestVersion;
- else if (hdfsLatestVersion != null) {
- latestVersion = hdfsLatestVersion;
+ if (curVer == null || curVer.isEmpty())
+ throw new IllegalStateException("Failed to determine Ignite version");
- hdfs = true;
- }
- }
+ log.info("Searching for Ignite release " + curVer);
- String newVersion = updateIgnite(latestVersion);
+ File release = findIgnite(folder, curVer);
- if (latestVersion != null && newVersion.equals(latestVersion)) {
- if (hdfs)
- return new Path(formatPath(props.igniteReleasesDir(), latestVersion));
- else
- return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion),
- formatPath(props.igniteReleasesDir(), latestVersion));
- }
- else {
- latestVersion = newVersion;
+ if (release == null) {
+ Path releaseOnHdfs = findIgnite(fs, props.igniteReleasesDir(), curVer);
+
+ if (releaseOnHdfs != null)
+ return releaseOnHdfs;
- return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion),
- formatPath(props.igniteReleasesDir(), latestVersion));
+ release = updateIgnite(curVer);
}
+
+ return IgniteYarnUtils.copyLocalToHdfs(fs, release.getAbsolutePath(),
+ props.igniteReleasesDir() + File.separator + release.getName());
}
/**
* @param folder Folder.
+ * @param curVer Ignite version.
* @return Ignite archives.
*/
- private List<String> findIgnites(File folder) {
+ private File findIgnite(File folder, String curVer) {
String[] files = folder.list();
- List<String> ignites = new ArrayList<>();
-
if (files != null) {
for (String fileName : files) {
- if (fileName.contains("gridgain-professional-") && fileName.endsWith(".zip"))
- ignites.add(fileName);
- }
- }
+ if (fileName.equals(igniteRelease(curVer))) {
+ log.info("Found local release at " + folder.getAbsolutePath());
- return ignites;
- }
-
- /**
- * @param files Files.
- * @return latest ignite version.
- */
- private String findLatestVersion(List<String> files) {
- String latestVersion = null;
-
- if (!files.isEmpty()) {
- if (files.size() == 1)
- latestVersion = parseVersion(files.get(0));
- else
- latestVersion = parseVersion(Collections.max(files, VersionComparator.INSTANCE));
+ return new File(folder, fileName);
+ }
+ }
}
- return latestVersion;
+ return null;
}
/**
* @param fs File system,
* @param folder Folder.
+ * @param curVer Ignite version.
* @return Ignite archives.
*/
- private List<String> findIgnites(FileSystem fs, String folder) {
+ private Path findIgnite(FileSystem fs, String folder, String curVer) {
FileStatus[] fileStatuses = null;
try {
@@ -158,35 +128,38 @@ public class IgniteProvider {
// Ignore. Folder doesn't exist.
}
catch (Exception e) {
- throw new RuntimeException("Couldnt get list files from hdfs.", e);
+ throw new RuntimeException("Couldn't get list files from hdfs.", e);
}
- List<String> ignites = new ArrayList<>();
-
if (fileStatuses != null) {
for (FileStatus file : fileStatuses) {
String fileName = file.getPath().getName();
- if (fileName.contains("gridgain-professional-") && fileName.endsWith(".zip"))
- ignites.add(fileName);
+ if (fileName.equals(igniteRelease(curVer))) {
+ log.info("Found HDFS release at " + file.getPath());
+
+ return file.getPath();
+ }
}
}
- return ignites;
+ return null;
}
/**
- * @param version Ignite version.
+ * @param igniteUrl Download link.
* @return Ignite.
*/
- public Path getIgnite(String version) throws Exception {
+ public Path getIgnite(String igniteUrl) throws Exception {
checkDownloadFolder();
// Download ignite.
- String fileName = downloadIgnite(version);
+ String fileName = downloadIgnite(igniteUrl);
Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName);
+ log.info("Using specified release at " + igniteUrl);
+
if (!fs.exists(dst))
fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst);
@@ -194,50 +167,50 @@ public class IgniteProvider {
}
/**
- * @param folder folder
- * @param version version
- * @return Path
+ * @return File name.
*/
- private static String formatPath(String folder, String version) {
- return folder + File.separator + "gridgain-professional-" + version + ".zip";
+ private static String igniteRelease(String version) {
+ return "apache-ignite-" + version + "-bin.zip";
}
/**
- * @param currentVersion The current latest version.
+ * @param curVer Ignite version.
* @return Current version if the current version is latest; new ignite version otherwise.
*/
- private String updateIgnite(String currentVersion) {
+ private File updateIgnite(String curVer) {
try {
- URL url;
-
- if (currentVersion == null)
- url = new URL(DOWNLOAD_LINK);
- else
- url = new URL(DOWNLOAD_LINK + "?version=" + currentVersion);
+ // Such as https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/2.7.0/apache-ignite-2.7.0-bin.zip
+ URL url = new URL(DOWNLOAD_LINK + curVer + "/" + igniteRelease(curVer));
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
int code = conn.getResponseCode();
- if (code == 200) {
- String redirectUrl = conn.getURL().toString();
-
- checkDownloadFolder();
+ String redirectUrl = conn.getURL().toString();
- FileOutputStream outFile = new FileOutputStream(props.igniteLocalWorkDir() + File.separator
- + fileName(redirectUrl));
-
- outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
+ if (code == 301 || code == 302) {
+ redirectUrl = conn.getHeaderField("Location");
- outFile.close();
+ conn.disconnect();
- return parseVersion(redirectUrl);
+ conn = (HttpURLConnection)new URL(redirectUrl).openConnection();
}
- else if (code == 304)
- // This version is latest.
- return currentVersion;
- else
+ else if (code != 200)
throw new RuntimeException("Got unexpected response code. Response code: " + code);
+
+ checkDownloadFolder();
+
+ File ignite = new File(props.igniteLocalWorkDir(), fileName(redirectUrl));
+
+ FileOutputStream outFile = new FileOutputStream(ignite);
+
+ outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);
+
+ outFile.close();
+
+ log.info("Found remote release at " + redirectUrl);
+
+ return ignite;
}
catch (IOException e) {
throw new RuntimeException("Failed update ignite.", e);
@@ -297,16 +270,6 @@ public class IgniteProvider {
/**
* @param url URL.
- * @return Ignite version.
- */
- private static String parseVersion(String url) {
- String[] split = url.split("-");
-
- return split[split.length - 1].replaceAll(".zip", "");
- }
-
- /**
- * @param url URL.
* @return File name.
*/
private static String fileName(String url) {
@@ -314,34 +277,4 @@ public class IgniteProvider {
return split[split.length - 1];
}
-
- /**
- * Ignite version comparator.
- */
- public static final class VersionComparator implements Comparator<String> {
- /** */
- public static final VersionComparator INSTANCE = new VersionComparator();
-
- /** */
- private VersionComparator() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public int compare(String f1, String f2) {
- if (f1.equals(f2))
- return 0;
-
- String[] ver1 = parseVersion(f1).split("\\.");
- String[] ver2 = parseVersion(f2).split("\\.");
-
- if (Integer.valueOf(ver1[0]) >= Integer.valueOf(ver2[0])
- && Integer.valueOf(ver1[1]) >= Integer.valueOf(ver2[1])
- && Integer.valueOf(ver1[2]) >= Integer.valueOf(ver2[2]))
-
- return 1;
- else
- return -1;
- }
- }
}
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
index 869a0d0..f9753e8 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
@@ -118,7 +118,7 @@ public class IgniteYarnClient {
String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
- if (tokRenewer == null || tokRenewer.length() == 0)
+ if (tokRenewer == null || tokRenewer.isEmpty())
throw new IOException("Master Kerberos principal for the RM is not set.");
log.info("Found RM principal: " + tokRenewer);
@@ -190,10 +190,12 @@ public class IgniteYarnClient {
private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception {
IgniteProvider provider = new IgniteProvider(props, fileSystem);
- if (props.igniteUrl() == null)
- return provider.getIgnite();
- else
- return provider.getIgnite(props.igniteUrl());
+ String igniteUrl = props.igniteUrl();
+
+ if (igniteUrl != null)
+ return provider.getIgnite(igniteUrl);
+
+ return provider.getIgnite();
}
/**
diff --git a/modules/yarn/src/main/resources/ignite.properties b/modules/yarn/src/main/resources/ignite.properties
new file mode 100644
index 0000000..57dcd33
--- /dev/null
+++ b/modules/yarn/src/main/resources/ignite.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ignite.version=2.7.0-SNAPSHOT
+ignite.build=0
+ignite.revision=DEV
+ignite.rel.date=01011970
+ignite.update.status.params=test=vfvfvskfkeievskjv
+ignite.update.notifier.enabled.by.default=false