You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@taverna.apache.org by st...@apache.org on 2016/02/13 14:55:27 UTC
[3/6] incubator-taverna-osgi git commit: TAVERNA-893 USe HTTPClient 4
and MessageDigest
TAVERNA-893 USe HTTPClient 4 and MessageDigest
Now supports all Java-supported message digests, including
SHA-1 and SHA-256
Local files supported via java.nio Path URIs.
No longer supported are classical URLs
registered with URLConnection, like ftp://
Project: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/commit/8a82204a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/tree/8a82204a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/diff/8a82204a
Branch: refs/heads/master
Commit: 8a82204a930e99c587f5d60c5df2318a2d8290fe
Parents: d6c43d6
Author: Stian Soiland-Reyes <st...@apache.org>
Authored: Fri Feb 12 17:47:47 2016 +0000
Committer: Stian Soiland-Reyes <st...@apache.org>
Committed: Fri Feb 12 17:47:47 2016 +0000
----------------------------------------------------------------------
taverna-download-impl/pom.xml | 23 ++-
.../download/impl/DownloadManagerImpl.java | 156 +++++++++++++------
.../download/impl/TestDownloadManagerImpl.java | 30 +++-
3 files changed, 147 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/pom.xml
----------------------------------------------------------------------
diff --git a/taverna-download-impl/pom.xml b/taverna-download-impl/pom.xml
index bc50c70..e81a1aa 100644
--- a/taverna-download-impl/pom.xml
+++ b/taverna-download-impl/pom.xml
@@ -16,7 +16,8 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.taverna.osgi</groupId>
@@ -45,21 +46,33 @@
<artifactId>taverna-download-api</artifactId>
<version>${project.parent.version}</version>
</dependency>
-
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons.io.version}</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient-osgi</artifactId>
+ <version>${apache.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <!-- Needed by httpclient-osgi -->
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-osgi</artifactId>
+ <version>${apache.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <!-- Needed by httpclient-osgi -->
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.2</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons.codec.version}</version>
</dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java b/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java
index 7292c8e..fa76723 100644
--- a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java
+++ b/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java
@@ -16,14 +16,25 @@
*/
package org.apache.taverna.download.impl;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystemNotFoundException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Locale;
+import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.FileUtils;
+import org.apache.http.client.fluent.Request;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.VersionInfo;
import org.apache.log4j.Logger;
import org.apache.taverna.download.DownloadException;
import org.apache.taverna.download.DownloadManager;
@@ -33,81 +44,128 @@ import org.apache.taverna.download.DownloadManager;
*/
public class DownloadManagerImpl implements DownloadManager {
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+
private static final int TIMEOUT = Integer.getInteger("taverna.download.timeout.seconds", 30) * 1000;
-
+
private static final Logger logger = Logger.getLogger(DownloadManagerImpl.class);
- public void download(URL source, File destination) throws DownloadException {
+ @Override
+ public void download(URI source, Path destination) throws DownloadException {
download(source, destination, null);
}
- public void download(URL source, File destination, String digestAlgorithm) throws DownloadException {
- // TODO Use MessageDigest when Java 7 available
- if (digestAlgorithm != null && !digestAlgorithm.equals("MD5")) {
- throw new IllegalArgumentException("Only MD5 supported");
- }
- URL digestSource = null;
+ @Override
+ public void download(URI source, Path destination, String digestAlgorithm) throws DownloadException {
+ URI digestSource = null;
if (digestAlgorithm != null) {
- try {
- digestSource = new URL(source.toString() + mapAlgorithmToFileExtension(digestAlgorithm));
- } catch (MalformedURLException e) {
- throw new DownloadException("Error creating digest URL", e);
- }
+ // Note: Will break with ?download=file.xml kind of URLs
+ digestSource = source.resolve(source.getPath() + mapAlgorithmToFileExtension(digestAlgorithm));
}
download(source, destination, digestAlgorithm, digestSource);
}
- public void download(URL source, File destination, String digestAlgorithm, URL digestSource)
+ public String getUserAgent() {
+ Package pack = getClass().getPackage();
+ String httpClientVersion = VersionInfo.getUserAgent("Apache-HttpClient", "org.apache.http.client",
+ Request.class);
+ return "Apache-Taverna-OSGi" + "/" + pack.getImplementationVersion() + " (" + httpClientVersion + ")";
+ }
+
+ @Override
+ public void download(URI source, Path destination, String digestAlgorithm, URI digestSource)
throws DownloadException {
- // TODO Use MessageDigest when Java 7 available
- if (digestAlgorithm != null && !digestAlgorithm.equals("MD5")) {
- throw new IllegalArgumentException("Only MD5 supported");
+
+ MessageDigest md = null;
+ if (digestAlgorithm != null) {
+ try {
+ md = MessageDigest.getInstance(digestAlgorithm);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalArgumentException("Unsupported digestAlgorithm: " + digestAlgorithm, e);
+ }
}
+
// download the file
- File tempFile;
+ Path tempFile;
try {
- tempFile = File.createTempFile("DownloadManager", "tmp");
- tempFile.deleteOnExit();
- logger.info(String.format("Downloading %1$s to %2$s", source, tempFile));
- FileUtils.copyURLToFile(source, tempFile, TIMEOUT, TIMEOUT);
- } catch (IOException e) {
- throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e);
+ tempFile = Files.createTempFile(destination.getParent(), "." + destination.getFileName(), ".tmp");
+ } catch (IOException e1) {
+ // perhaps a permission problem?
+ throw new DownloadException("Can't create temporary file in folder " + destination.getParent(), e1);
}
+ logger.info(String.format("Downloading %1$s to %2$s", source, tempFile));
+ downloadToFile(source, tempFile);
+
if (digestSource != null) {
// download the digest file
- File digestFile;
+ String expectedDigest;
+ expectedDigest = downloadHash(digestSource).trim().toLowerCase(Locale.ROOT);
+ // check if the digest matches
try {
- digestFile = File.createTempFile("DownloadManager", "tmp");
- digestFile.deleteOnExit();
- logger.info(String.format("Downloading %1$s to %2$s", digestSource, digestFile));
- FileUtils.copyURLToFile(digestSource, digestFile, TIMEOUT, TIMEOUT);
- } catch (IOException e) {
- throw new DownloadException(String.format("Error checking digest for %1$s.", source), e);
- }
- // check the digest matches
- try {
- String digestString1 = DigestUtils.md5Hex(new FileInputStream(tempFile));
- String digestString2 = FileUtils.readFileToString(digestFile);
- if (!digestString1.equals(digestString2)) {
- throw new DownloadException(String.format(
- "Error downloading file: digsests not equal. (%1$s != %2$s)",
- digestString1, digestString2));
+ try (InputStream s = Files.newInputStream(tempFile)) {
+ DigestUtils.updateDigest(md, s);
+ String actualDigest = Hex.encodeHexString(md.digest());
+ if (!actualDigest.equals(expectedDigest)) {
+ throw new DownloadException(
+ String.format("Error downloading file: checksum mismatch (%1$s != %2$s)",
+ actualDigest, expectedDigest));
+ }
}
} catch (IOException e) {
- throw new DownloadException(String.format("Error checking digest for %1$s", destination),
- e);
+ throw new DownloadException(String.format("Error checking digest for %1$s", destination), e);
}
}
- // copy file to destination
+ // All fine, move to destination
try {
logger.info(String.format("Copying %1$s to %2$s", tempFile, destination));
- FileUtils.copyFile(tempFile, destination);
+ Files.move(tempFile, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e);
}
}
+ private String downloadHash(URI source) throws DownloadException {
+ try {
+ // We want to handle http/https with HTTPClient
+ if (source.getScheme().equalsIgnoreCase("http") || source.getScheme().equalsIgnoreCase("https")) {
+ logger.info("Downloading checksum " + source);
+ return Request.Get(source).userAgent(getUserAgent()).connectTimeout(TIMEOUT).socketTimeout(TIMEOUT).execute()
+ .returnContent().asString(StandardCharsets.ISO_8859_1);
+ } else {
+ // Try as a supported Path, e.g. file: or relative path
+ try {
+ Path path = Paths.get(source);
+ return Files.readAllLines(path, StandardCharsets.ISO_8859_1).get(0);
+ } catch (FileSystemNotFoundException e) {
+ throw new DownloadException("Unsupported URL scheme: " + source.getScheme());
+ }
+ }
+ } catch (IOException e) {
+ throw new DownloadException(String.format("Error downloading %1$s", source), e);
+ }
+ }
+
+ private void downloadToFile(URI source, Path destination) throws DownloadException {
+ try {
+ // We want to handle http/https with HTTPClient
+ if (source.getScheme().equalsIgnoreCase("http") || source.getScheme().equalsIgnoreCase("https")) {
+ Request.Get(source).userAgent(getUserAgent()).connectTimeout(TIMEOUT).socketTimeout(TIMEOUT).execute()
+ .saveContent(destination.toFile());
+ } else {
+ // Try as a supported Path, e.g. file: or relative path
+ try {
+ Path path = Paths.get(source);
+ Files.copy(path, destination, StandardCopyOption.REPLACE_EXISTING);
+ } catch (FileSystemNotFoundException e) {
+ throw new DownloadException("Unsupported URL scheme: " + source.getScheme());
+ }
+ }
+ } catch (IOException e) {
+ throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e);
+ }
+ }
+
private String mapAlgorithmToFileExtension(String algorithm) {
return "." + algorithm.toLowerCase().replaceAll("-", "");
}
http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java b/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java
index a260c59..fbe2dcb 100644
--- a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java
+++ b/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java
@@ -3,7 +3,7 @@ package org.apache.taverna.download.impl;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.junit.Assert.assertEquals;
-import java.net.URL;
+import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -15,7 +15,7 @@ public class TestDownloadManagerImpl {
/**
* This test should remain @Ignored
- * as it relies on a third-party web site
+ * as it relies on a web site
* and should not break the build.
*
* Verifies TAVERNA-893
@@ -28,13 +28,13 @@ public class TestDownloadManagerImpl {
Path pomFile = Files.createTempFile("test", ".pom");
// NOTE: The below URL is a Taverna 2 POM - not related to
// taverna-plugin-impl
- URL wrongURL = new URL("http://192.185.115.65/~diana/DIANA_plugin_updated/test-plugins/gr/dianatools/diana.services-activity/1.0-SNAPSHOT/diana.services-activity-1.0-SNAPSHOT.pom");
+ URI wrongURL = URI.create("http://192.185.115.65/~diana/DIANA_plugin_updated/test-plugins/gr/dianatools/diana.services-activity/1.0-SNAPSHOT/diana.services-activity-1.0-SNAPSHOT.pom");
// With this we get an exception - but only because the
// 300 erorr page fails the MD5 check
//dl.download(wrongURL, pomFile.toFile(), "MD5");
// so we'll try without hashsum
- dl.download(wrongURL, pomFile.toFile());
+ dl.download(wrongURL, pomFile);
}
@@ -47,7 +47,7 @@ public class TestDownloadManagerImpl {
*/
@Test
public void downloadLocalFile() throws Exception {
- Path example = Files.createTempFile("example", ".txt");
+ Path example = Files.createTempFile("test", ".txt");
Files.write(example, "Hello world".getBytes(US_ASCII)); // No newline
Path exampleMd5 = example.resolveSibling(example.getFileName() + ".md5");
// stain@biggie:~$ echo -n "Hello world"|md5sum
@@ -58,11 +58,25 @@ public class TestDownloadManagerImpl {
Path toFile = Files.createTempFile("downloaded", ".txt");
- dl.download(example.toUri().toURL(), toFile.toFile(), "MD5");
+ dl.download(example.toUri(), toFile, "MD5");
String hello = Files.readAllLines(toFile, US_ASCII).get(0);
- assertEquals("Hello world", hello);
-
+ assertEquals("Hello world", hello);
+ }
+
+ /**
+ * This test should remain @Ignored
+ * as it relies on a web site
+ * and should not break the build.
+ *
+ */
+ @Ignore
+ @Test
+ public void downloadPomSha1() throws Exception {
+ Path destination = Files.createTempFile("test", "pom");
+ URI source = URI.create("https://repo.maven.apache.org/maven2/org/apache/apache/17/apache-17.pom");
+ DownloadManagerImpl dl = new DownloadManagerImpl();
+ dl.download(source, destination, "SHA-1");
}
}