You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2015/04/09 17:23:13 UTC
[3/9] incubator-ignite git commit: # IGNITE-661: Removed FTP scanner
and abstracted scanners into separate configurable entity.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java
deleted file mode 100644
index dc9044e..0000000
--- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.deployment.uri.scanners.ftp;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.deployment.uri.scanners.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-/**
- * FTP scanner scans directory for new files. Scanned directory defined in URI.
- * Scanner doesn't search files in subfolders.
- */
-public class GridUriDeploymentFtpScanner extends GridUriDeploymentScanner {
- /** */
- private static final long UNKNOWN_FILE_TSTAMP = -1;
-
- /** */
- private final GridUriDeploymentFtpConfiguration cfg;
-
- /** Cache of found files to check if any of it has been updated. */
- private Map<GridUriDeploymentFtpFile, Long> cache = new HashMap<>();
-
- /**
- * @param gridName Grid instance name.
- * @param uri FTP URI.
- * @param deployDir FTP directory.
- * @param freq Scanner frequency.
- * @param filter Scanner filter.
- * @param lsnr Deployment listener.
- * @param log Logger to use.
- */
- public GridUriDeploymentFtpScanner(
- String gridName,
- URI uri,
- File deployDir,
- long freq,
- FilenameFilter filter,
- GridUriDeploymentScannerListener lsnr,
- IgniteLogger log) {
- super(gridName, uri, deployDir, freq, filter, lsnr, log);
-
- cfg = initializeFtpConfiguration(uri);
- }
-
- /**
- * @param uri FTP URI.
- * @return FTP configuration.
- */
- private GridUriDeploymentFtpConfiguration initializeFtpConfiguration(URI uri) {
- assert "ftp".equals(uri.getScheme());
-
- GridUriDeploymentFtpConfiguration cfg = new GridUriDeploymentFtpConfiguration();
-
- String userInfo = uri.getUserInfo();
- String username = null;
- String pswd = null;
-
- if (userInfo != null) {
- String[] arr = userInfo.split(";");
-
- if (arr != null && arr.length > 0)
- for (String el : arr)
- if (el.startsWith("freq=")) {
- // No-op.
- }
- else if (el.indexOf(':') != -1) {
- int idx = el.indexOf(':');
-
- username = el.substring(0, idx);
- pswd = el.substring(idx + 1);
- }
- else
- username = el;
- }
-
- // Username and password must be defined in URI.
- if (username == null)
- throw new IgniteException("Username has not been provided.");
-
- if (pswd == null)
- throw new IgniteException("Password has not been provided.");
-
- cfg.setHost(uri.getHost());
- cfg.setPort(uri.getPort());
- cfg.setUsername(username);
- cfg.setPassword(pswd);
- cfg.setDirectory(uri.getPath());
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void process() {
- Collection<GridUriDeploymentFtpFile> foundFiles = U.newHashSet(cache.size());
-
- long start = U.currentTimeMillis();
-
- processFtp(foundFiles);
-
- if (getLogger().isDebugEnabled())
- getLogger().debug("FTP scanner time in milliseconds: " + (U.currentTimeMillis() - start));
-
- if (!isFirstScan()) {
- Collection<GridUriDeploymentFtpFile> delFiles = new HashSet<>(cache.keySet());
-
- delFiles.removeAll(foundFiles);
-
- if (!delFiles.isEmpty()) {
- List<String> uris = new ArrayList<>();
-
- for (GridUriDeploymentFtpFile file : delFiles) {
- Long tstamp = cache.get(file);
-
- // Ignore files in cache w/o timestamp.
- if (tstamp != null && tstamp != UNKNOWN_FILE_TSTAMP)
- uris.add(getFileUri(file.getName()));
- }
-
- cache.keySet().removeAll(delFiles);
-
- getListener().onDeletedFiles(uris);
- }
- }
- }
-
- /**
- * @param files File to process.
- */
- @SuppressWarnings({"UnusedCatchParameter"})
- private void processFtp(Collection<GridUriDeploymentFtpFile> files) {
- GridUriDeploymentFtpClient ftp = new GridUriDeploymentFtpClient(cfg, getLogger());
-
- try {
- ftp.connect();
-
- for (GridUriDeploymentFtpFile file : ftp.getFiles()) {
- String fileName = file.getName();
-
- if (getFilter().accept(null, fileName.toLowerCase()) && file.isFile()) {
- files.add(file);
-
- Long lastModified = cache.get(file);
-
- Calendar fileTstamp = file.getTimestamp();
-
- if (fileTstamp == null) {
- if (lastModified == null) {
- // Add new file in cache to avoid print warning every time.
- cache.put(file, UNKNOWN_FILE_TSTAMP);
-
- U.warn(getLogger(), "File with unknown timestamp will be ignored " +
- "(check FTP server configuration): " + file);
- }
- }
- // If file is new or has been modified.
- else if (lastModified == null || lastModified != fileTstamp.getTimeInMillis()) {
- cache.put(file, fileTstamp.getTimeInMillis());
-
- if (getLogger().isDebugEnabled())
- getLogger().debug("Discovered deployment file or directory: " + file);
-
- try {
- File diskFile = createTempFile(fileName, getDeployDirectory());
-
- ftp.downloadToFile(file, diskFile);
-
- String fileUri = getFileUri(fileName);
-
- // Delete file when JVM stopped.
- diskFile.deleteOnExit();
-
- // Deployment SPI apply.
- // NOTE: If SPI listener blocks then FTP connection may be closed by timeout.
- getListener().onNewOrUpdatedFile(diskFile, fileUri, fileTstamp.getTimeInMillis());
- }
- catch (IOException e) {
- U.error(getLogger(), "Failed to download file from FTP server: " + fileName, e);
- }
- }
- }
- }
- }
- catch (GridUriDeploymentFtpException e) {
- if (!isCancelled()) {
- String maskedUri = getUri() != null ? U.hidePassword(getUri().toString()) : null;
-
- if (e.hasCause(ConnectException.class))
- LT.warn(getLogger(), e, "Failed to connect to FTP server (connection refused): " + maskedUri);
-
- else if (e.hasCause(UnknownHostException.class))
- LT.warn(getLogger(), e, "Failed to connect to FTP server (host is unknown): " + maskedUri);
-
- else
- U.error(getLogger(), "Failed to get files from FTP server: " + maskedUri, e);
- }
- }
- finally {
- try {
- ftp.close();
- }
- catch (GridUriDeploymentFtpException e) {
- if (!isCancelled())
- U.error(getLogger(), "Failed to close FTP client.", e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridUriDeploymentFtpScanner.class, this,
- "uri", getUri() != null ? U.hidePassword(getUri().toString()) : null,
- "freq", getFrequency(),
- "deployDir", getDeployDirectory());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
deleted file mode 100644
index ca29005..0000000
--- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains FTP scanner for URI deployer SPI.
- */
-package org.apache.ignite.spi.deployment.uri.scanners.ftp;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java
deleted file mode 100644
index 49ac343..0000000
--- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.deployment.uri.scanners.http;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.tostring.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.*;
-import org.apache.ignite.spi.deployment.uri.scanners.*;
-import org.jetbrains.annotations.*;
-import org.w3c.dom.*;
-import org.w3c.tidy.*;
-
-import javax.net.ssl.*;
-import java.io.*;
-import java.net.*;
-import java.security.*;
-import java.security.cert.*;
-import java.util.*;
-
-/**
- * URI deployment HTTP scanner.
- */
-public class GridUriDeploymentHttpScanner extends GridUriDeploymentScanner {
- /** Secure socket protocol to use. */
- private static final String PROTOCOL = "TLS";
-
- /** */
- @GridToStringExclude
- private URL scanDir;
-
- /** Cache of found files to check if any of it has been updated. */
- private Map<String, Long> tstampCache = new HashMap<>();
-
- /** */
- @GridToStringExclude
- private final Tidy tidy;
-
- /** Outgoing data SSL socket factory. */
- private SSLSocketFactory sockFactory;
-
- /**
- * @param gridName Grid instance name.
- * @param uri HTTP URI.
- * @param deployDir Deployment directory.
- * @param freq Scanner frequency.
- * @param filter Filename filter.
- * @param lsnr Deployment listener.
- * @param log Logger to use.
- * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error.
- */
- public GridUriDeploymentHttpScanner(
- String gridName,
- URI uri,
- File deployDir,
- long freq,
- FilenameFilter filter,
- GridUriDeploymentScannerListener lsnr,
- IgniteLogger log) throws IgniteSpiException {
- super(gridName, uri, deployDir, freq, filter, lsnr, log);
-
- initialize(uri);
-
- tidy = new Tidy();
-
- tidy.setQuiet(true);
- tidy.setOnlyErrors(true);
- tidy.setShowWarnings(false);
- tidy.setInputEncoding("UTF8");
- tidy.setOutputEncoding("UTF8");
- }
-
- /**
- * @param uri HTTP URI.
- * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error.
- */
- private void initialize(URI uri) throws IgniteSpiException {
- assert "http".equals(uri.getScheme()) || "https".equals(uri.getScheme());
-
- try {
- scanDir = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath());
- }
- catch (MalformedURLException e) {
- scanDir = null;
-
- throw new IgniteSpiException("Wrong value for scanned HTTP directory with URI: " + uri, e);
- }
-
- try {
- if ("https".equals(uri.getScheme())) {
- // Set up socket factory to do authentication.
- SSLContext ctx = SSLContext.getInstance(PROTOCOL);
-
- ctx.init(null, getTrustManagers(), null);
-
- sockFactory = ctx.getSocketFactory();
- }
- }
- catch (NoSuchAlgorithmException e) {
- throw new IgniteSpiException("Failed to initialize SSL context. URI: " + uri, e);
- }
- catch (KeyManagementException e) {
- throw new IgniteSpiException("Failed to initialize SSL context. URI:" + uri, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void process() {
- Collection<String> foundFiles = U.newHashSet(tstampCache.size());
-
- long start = U.currentTimeMillis();
-
- processHttp(foundFiles);
-
- if (getLogger().isDebugEnabled())
- getLogger().debug("HTTP scanner time in ms: " + (U.currentTimeMillis() - start));
-
- if (!isFirstScan()) {
- Collection<String> deletedFiles = new HashSet<>(tstampCache.keySet());
-
- deletedFiles.removeAll(foundFiles);
-
- if (!deletedFiles.isEmpty()) {
- List<String> uris = new ArrayList<>();
-
- for (String file : deletedFiles) {
- uris.add(getFileUri(getFileName(file)));
- }
-
- tstampCache.keySet().removeAll(deletedFiles);
-
- getListener().onDeletedFiles(uris);
- }
- }
- }
-
- /**
- * @param files Files to process.
- */
- private void processHttp(Collection<String> files) {
- Set<String> urls = getUrls(scanDir);
-
- for (String url : urls) {
- String fileName = getFileName(url);
-
- if (getFilter().accept(null, fileName)) {
- files.add(url);
-
- Long lastModified = tstampCache.get(url);
-
- InputStream in = null;
- OutputStream out = null;
-
- File file = null;
-
- try {
- URLConnection conn = new URL(url).openConnection();
-
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection httpsConn = (HttpsURLConnection)conn;
-
- httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier());
-
- assert sockFactory != null;
-
- // Initialize socket factory.
- httpsConn.setSSLSocketFactory(sockFactory);
- }
-
- if (lastModified != null)
- conn.setIfModifiedSince(lastModified);
-
- in = conn.getInputStream();
-
- long rcvLastModified = conn.getLastModified();
-
- if (in == null || lastModified != null && (lastModified == rcvLastModified ||
- conn instanceof HttpURLConnection &&
- ((HttpURLConnection)conn).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED))
- continue;
-
- tstampCache.put(url, rcvLastModified);
-
- lastModified = rcvLastModified;
-
- if (getLogger().isDebugEnabled()) {
- getLogger().debug("Discovered deployment file or directory: " +
- U.hidePassword(url));
- }
-
- file = createTempFile(fileName, getDeployDirectory());
-
- // Delete file when JVM stopped.
- file.deleteOnExit();
-
- out = new FileOutputStream(file);
-
- U.copy(in, out);
- }
- catch (IOException e) {
- if (!isCancelled()) {
- if (X.hasCause(e, ConnectException.class)) {
- LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " +
- U.hidePassword(url));
- }
- else if (X.hasCause(e, UnknownHostException.class)) {
- LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " +
- U.hidePassword(url));
- }
- else
- U.error(getLogger(), "Failed to save file: " + fileName, e);
- }
- }
- finally {
- U.closeQuiet(in);
- U.closeQuiet(out);
- }
-
- if (file != null && file.exists() && file.length() > 0)
- getListener().onNewOrUpdatedFile(file, getFileUri(fileName), lastModified);
- }
- }
- }
-
- /**
- * @param node XML element node.
- * @param res Set of URLs in string format to populate.
- * @param baseUrl Base URL.
- */
- @SuppressWarnings( {"UnusedCatchParameter", "UnnecessaryFullyQualifiedName"})
- private void findReferences(org.w3c.dom.Node node, Set<String> res, URL baseUrl) {
- if (node instanceof Element && "a".equals(node.getNodeName().toLowerCase())) {
- Element element = (Element)node;
-
- String href = element.getAttribute("href");
-
- if (href != null && !href.isEmpty()) {
- URL url = null;
-
- try {
- url = new URL(href);
- }
- catch (MalformedURLException e) {
- try {
- url = new URL(baseUrl.getProtocol(), baseUrl.getHost(), baseUrl.getPort(),
- href.charAt(0) == '/' ? href : baseUrl.getFile() + '/' + href);
- }
- catch (MalformedURLException e1) {
- U.error(getLogger(), "Skipping bad URL: " + url, e1);
- }
- }
-
- if (url != null)
- res.add(url.toString());
- }
- }
-
- NodeList childNodes = node.getChildNodes();
-
- for (int i = 0; i < childNodes.getLength(); i++)
- findReferences(childNodes.item(i), res, baseUrl);
- }
-
- /**
- * @param url Base URL.
- * @return Set of referenced URLs in string format.
- */
- private Set<String> getUrls(URL url) {
- assert url != null;
-
- InputStream in = null;
-
- Set<String> urls = new HashSet<>();
-
- Document dom = null;
-
- try {
- URLConnection conn = url.openConnection();
-
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection httpsConn = (HttpsURLConnection)conn;
-
- httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier());
-
- assert sockFactory != null;
-
- // Initialize socket factory.
- httpsConn.setSSLSocketFactory(sockFactory);
- }
-
- in = conn.getInputStream();
-
- if (in == null)
- throw new IOException("Failed to open connection: " + U.hidePassword(url.toString()));
-
- dom = tidy.parseDOM(in, null);
- }
- catch (IOException e) {
- if (!isCancelled()) {
- if (X.hasCause(e, ConnectException.class)) {
- LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " +
- U.hidePassword(url.toString()));
- }
- else if (X.hasCause(e, UnknownHostException.class)) {
- LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " +
- U.hidePassword(url.toString()));
- }
- else
- U.error(getLogger(), "Failed to get HTML page: " + U.hidePassword(url.toString()), e);
- }
- }
- finally{
- U.closeQuiet(in);
- }
-
- if (dom != null)
- findReferences(dom, urls, url);
-
- return urls;
- }
-
- /**
- * @param url Base URL string format.
- * @return File name extracted from {@code url} string format.
- */
- private String getFileName(String url) {
- assert url != null;
-
- return url.substring(url.lastIndexOf('/') + 1);
- }
-
- /**
- * Construct array with one trust manager which don't reject input certificates.
- *
- * @return Array with one X509TrustManager implementation of trust manager.
- */
- private TrustManager[] getTrustManagers() {
- return new TrustManager[]{
- new X509TrustManager() {
- /** {@inheritDoc} */
- @Nullable @Override public X509Certificate[] getAcceptedIssuers() { return null; }
-
- /** {@inheritDoc} */
- @Override public void checkClientTrusted(X509Certificate[] certs, String authType) {
- StringBuilder buf = new StringBuilder();
-
- buf.append("Trust manager handle client certificates [authType=");
- buf.append(authType);
- buf.append(", certificates=");
-
- for (X509Certificate cert : certs) {
- buf.append("{type=");
- buf.append(cert.getType());
- buf.append(", principalName=");
- buf.append(cert.getSubjectX500Principal().getName());
- buf.append('}');
- }
-
- buf.append(']');
-
- if (getLogger().isDebugEnabled())
- getLogger().debug(buf.toString());
- }
-
- /** {@inheritDoc} */
- @Override public void checkServerTrusted(X509Certificate[] certs, String authType) {
- StringBuilder buf = new StringBuilder();
-
- buf.append("Trust manager handle server certificates [authType=");
- buf.append(authType);
- buf.append(", certificates=");
-
- for (X509Certificate cert : certs) {
- buf.append("{type=");
- buf.append(cert.getType());
- buf.append(", principalName=");
- buf.append(cert.getSubjectX500Principal().getName());
- buf.append('}');
- }
-
- buf.append(']');
-
- if (getLogger().isDebugEnabled())
- getLogger().debug(buf.toString());
- }
- }
- };
- }
-
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridUriDeploymentHttpScanner.class, this,
- "scanDir", scanDir != null ? U.hidePassword(scanDir.toString()) : null);
- }
-
- /**
- * Verifier always return successful result for any host.
- */
- private static class DeploymentHostnameVerifier implements HostnameVerifier {
- /** {@inheritDoc} */
- @Override public boolean verify(String hostname, SSLSession ses) {
- // Remote host trusted by default.
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
new file mode 100644
index 0000000..002b2e1
--- /dev/null
+++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.deployment.uri.scanners.http;
+
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.*;
+import org.apache.ignite.spi.deployment.uri.scanners.*;
+import org.jetbrains.annotations.*;
+import org.w3c.dom.*;
+import org.w3c.tidy.*;
+
+import javax.net.ssl.*;
+import java.io.*;
+import java.net.*;
+import java.security.*;
+import java.security.cert.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * URI deployment HTTP scanner.
+ */
+public class UriDeploymentHttpScanner implements UriDeploymentScanner {
+ /** Default scan frequency. */
+ private static final int DFLT_SCAN_FREQ = 300000;
+
+ /** Secure socket protocol to use. */
+ private static final String PROTOCOL = "TLS";
+
+ /** Per-URI contexts. */
+ private final ConcurrentHashMap<URI, URIContext> uriCtxs = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public boolean acceptsURI(URI uri) {
+ String proto = uri.getScheme().toLowerCase();
+
+ return "http".equals(proto) || "https".equals(proto);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void scan(UriDeploymentScannerContext scanCtx) {
+ URI uri = scanCtx.getUri();
+
+ URIContext uriCtx = uriCtxs.get(uri);
+
+ if (uriCtx == null) {
+ uriCtx = createUriContext(uri, scanCtx);
+
+ URIContext oldUriCtx = uriCtxs.putIfAbsent(uri, uriCtx);
+
+ if (oldUriCtx != null)
+ uriCtx = oldUriCtx;
+ }
+
+ uriCtx.scan(scanCtx);
+ }
+
+ /**
+ * Create context for the given URI.
+ *
+ * @param uri URI.
+ * @param scanCtx Scanner context.
+ * @return URI context.
+ */
+ private URIContext createUriContext(URI uri, final UriDeploymentScannerContext scanCtx) {
+ assert "http".equals(uri.getScheme()) || "https".equals(uri.getScheme());
+
+ URL scanDir;
+
+ try {
+ scanDir = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath());
+ }
+ catch (MalformedURLException e) {
+ throw new IgniteSpiException("Wrong value for scanned HTTP directory with URI: " + uri, e);
+ }
+
+ SSLSocketFactory sockFactory = null;
+
+ try {
+ if ("https".equals(uri.getScheme())) {
+ // Set up socket factory to do authentication.
+ SSLContext ctx = SSLContext.getInstance(PROTOCOL);
+
+ ctx.init(null, getTrustManagers(scanCtx), null);
+
+ sockFactory = ctx.getSocketFactory();
+ }
+ }
+ catch (NoSuchAlgorithmException e) {
+ throw new IgniteSpiException("Failed to initialize SSL context. URI: " + uri, e);
+ }
+ catch (KeyManagementException e) {
+ throw new IgniteSpiException("Failed to initialize SSL context. URI:" + uri, e);
+ }
+
+ return new URIContext(scanDir, sockFactory);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDefaultScanFrequency() {
+ return DFLT_SCAN_FREQ;
+ }
+
+ /**
+ * Construct array with one trust manager which don't reject input certificates.
+ *
+ * @param scanCtx context.
+ * @return Array with one X509TrustManager implementation of trust manager.
+ */
+ private static TrustManager[] getTrustManagers(final UriDeploymentScannerContext scanCtx) {
+ return new TrustManager[]{
+ new X509TrustManager() {
+ /** {@inheritDoc} */
+ @Nullable
+ @Override public X509Certificate[] getAcceptedIssuers() { return null; }
+
+ /** {@inheritDoc} */
+ @Override public void checkClientTrusted(X509Certificate[] certs, String authType) {
+ StringBuilder buf = new StringBuilder();
+
+ buf.append("Trust manager handle client certificates [authType=");
+ buf.append(authType);
+ buf.append(", certificates=");
+
+ for (X509Certificate cert : certs) {
+ buf.append("{type=");
+ buf.append(cert.getType());
+ buf.append(", principalName=");
+ buf.append(cert.getSubjectX500Principal().getName());
+ buf.append('}');
+ }
+
+ buf.append(']');
+
+ if (scanCtx.getLogger().isDebugEnabled())
+ scanCtx.getLogger().debug(buf.toString());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void checkServerTrusted(X509Certificate[] certs, String authType) {
+ StringBuilder buf = new StringBuilder();
+
+ buf.append("Trust manager handle server certificates [authType=");
+ buf.append(authType);
+ buf.append(", certificates=");
+
+ for (X509Certificate cert : certs) {
+ buf.append("{type=");
+ buf.append(cert.getType());
+ buf.append(", principalName=");
+ buf.append(cert.getSubjectX500Principal().getName());
+ buf.append('}');
+ }
+
+ buf.append(']');
+
+ if (scanCtx.getLogger().isDebugEnabled())
+ scanCtx.getLogger().debug(buf.toString());
+ }
+ }
+ };
+ }
+
+ /**
+ * URI context.
+ */
+ private class URIContext {
+ /** */
+ private final URL scanDir;
+
+ /** Outgoing data SSL socket factory. */
+ private final SSLSocketFactory sockFactory;
+
+ /** */
+ private final Tidy tidy;
+
+ /** Cache of found files to check if any of it has been updated. */
+ private final Map<String, Long> tstampCache = new HashMap<>();
+
+ /**
+ * Constructor.
+ *
+ * @param scanDir Scan directory.
+ * @param sockFactory Socket factory.
+ */
+ public URIContext(URL scanDir, SSLSocketFactory sockFactory) {
+ this.scanDir = scanDir;
+ this.sockFactory = sockFactory;
+
+ tidy = new Tidy();
+
+ tidy.setQuiet(true);
+ tidy.setOnlyErrors(true);
+ tidy.setShowWarnings(false);
+ tidy.setInputEncoding("UTF8");
+ tidy.setOutputEncoding("UTF8");
+ }
+
+ /**
+ * Perform scan.
+ *
+ * @param scanCtx Scan context.
+ */
+ private void scan(UriDeploymentScannerContext scanCtx) {
+ Collection<String> foundFiles = U.newHashSet(tstampCache.size());
+
+ long start = U.currentTimeMillis();
+
+ processHttp(foundFiles, scanCtx);
+
+ if (scanCtx.getLogger().isDebugEnabled())
+ scanCtx.getLogger().debug("HTTP scanner time in ms: " + (U.currentTimeMillis() - start));
+
+ if (!scanCtx.isFirstScan()) {
+ Collection<String> deletedFiles = new HashSet<>(tstampCache.keySet());
+
+ deletedFiles.removeAll(foundFiles);
+
+ if (!deletedFiles.isEmpty()) {
+ List<String> uris = new ArrayList<>();
+
+ for (String file : deletedFiles)
+ uris.add(getFileUri(fileName(file), scanCtx));
+
+ tstampCache.keySet().removeAll(deletedFiles);
+
+ scanCtx.getListener().onDeletedFiles(uris);
+ }
+ }
+ }
+
+ /**
+ * @param files Files to process.
+ * @param scanCtx Scan context.
+ */
+ @SuppressWarnings("unchecked")
+ private void processHttp(Collection<String> files, UriDeploymentScannerContext scanCtx) {
+ Set<String> urls = getUrls(scanDir, scanCtx);
+
+ for (String url : urls) {
+ String fileName = fileName(url);
+
+ if (scanCtx.getFilter().accept(null, fileName)) {
+ files.add(url);
+
+ Long lastModified = tstampCache.get(url);
+
+ InputStream in = null;
+ OutputStream out = null;
+
+ File file = null;
+
+ try {
+ URLConnection conn = new URL(url).openConnection();
+
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection httpsConn = (HttpsURLConnection)conn;
+
+ httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier());
+
+ assert sockFactory != null;
+
+ // Initialize socket factory.
+ httpsConn.setSSLSocketFactory(sockFactory);
+ }
+
+ if (lastModified != null)
+ conn.setIfModifiedSince(lastModified);
+
+ in = conn.getInputStream();
+
+ long rcvLastModified = conn.getLastModified();
+
+ if (in == null || lastModified != null && (lastModified == rcvLastModified ||
+ conn instanceof HttpURLConnection &&
+ ((HttpURLConnection)conn).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED))
+ continue;
+
+ tstampCache.put(url, rcvLastModified);
+
+ lastModified = rcvLastModified;
+
+ if (scanCtx.getLogger().isDebugEnabled()) {
+ scanCtx.getLogger().debug("Discovered deployment file or directory: " +
+ U.hidePassword(url));
+ }
+
+ file = scanCtx.createTempFile(fileName, scanCtx.getDeployDirectory());
+
+ // Delete file when JVM stopped.
+ file.deleteOnExit();
+
+ out = new FileOutputStream(file);
+
+ U.copy(in, out);
+ }
+ catch (IOException e) {
+ if (!scanCtx.isCancelled()) {
+ if (X.hasCause(e, ConnectException.class)) {
+ LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
+ "(connection refused): " + U.hidePassword(url));
+ }
+ else if (X.hasCause(e, UnknownHostException.class)) {
+ LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " +
+ "(host is unknown): " + U.hidePassword(url));
+ }
+ else
+ U.error(scanCtx.getLogger(), "Failed to save file: " + fileName, e);
+ }
+ }
+ finally {
+ U.closeQuiet(in);
+ U.closeQuiet(out);
+ }
+
+ if (file != null && file.exists() && file.length() > 0)
+ scanCtx.getListener().onNewOrUpdatedFile(file, getFileUri(fileName, scanCtx), lastModified);
+ }
+ }
+ }
+
+ /**
+ * @param url Base URL.
+ * @param scanCtx Scan context.
+ * @return Set of referenced URLs in string format.
+ */
+ @SuppressWarnings("unchecked")
+ private Set<String> getUrls(URL url, UriDeploymentScannerContext scanCtx) {
+ assert url != null;
+
+ InputStream in = null;
+
+ Set<String> urls = new HashSet<>();
+
+ Document dom = null;
+
+ try {
+ URLConnection conn = url.openConnection();
+
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection httpsConn = (HttpsURLConnection)conn;
+
+ httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier());
+
+ assert sockFactory != null;
+
+ // Initialize socket factory.
+ httpsConn.setSSLSocketFactory(sockFactory);
+ }
+
+ in = conn.getInputStream();
+
+ if (in == null)
+ throw new IOException("Failed to open connection: " + U.hidePassword(url.toString()));
+
+ dom = tidy.parseDOM(in, null);
+ }
+ catch (IOException e) {
+ if (!scanCtx.isCancelled()) {
+ if (X.hasCause(e, ConnectException.class)) {
+ LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (connection refused): " +
+ U.hidePassword(url.toString()));
+ }
+ else if (X.hasCause(e, UnknownHostException.class)) {
+ LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (host is unknown): " +
+ U.hidePassword(url.toString()));
+ }
+ else
+ U.error(scanCtx.getLogger(), "Failed to get HTML page: " + U.hidePassword(url.toString()), e);
+ }
+ }
+ finally{
+ U.closeQuiet(in);
+ }
+
+ if (dom != null)
+ findReferences(dom, urls, url, scanCtx);
+
+ return urls;
+ }
+
+ /**
+ * @param node XML element node.
+ * @param res Set of URLs in string format to populate.
+ * @param baseUrl Base URL.
+ * @param scanCtx Scan context.
+ */
+ @SuppressWarnings( {"UnusedCatchParameter", "UnnecessaryFullyQualifiedName"})
+ private void findReferences(org.w3c.dom.Node node, Set<String> res, URL baseUrl,
+ UriDeploymentScannerContext scanCtx) {
+ if (node instanceof Element && "a".equals(node.getNodeName().toLowerCase())) {
+ Element element = (Element)node;
+
+ String href = element.getAttribute("href");
+
+ if (href != null && !href.isEmpty()) {
+ URL url = null;
+
+ try {
+ url = new URL(href);
+ }
+ catch (MalformedURLException e) {
+ try {
+ url = new URL(baseUrl.getProtocol(), baseUrl.getHost(), baseUrl.getPort(),
+ href.charAt(0) == '/' ? href : baseUrl.getFile() + '/' + href);
+ }
+ catch (MalformedURLException e1) {
+ U.error(scanCtx.getLogger(), "Skipping bad URL: " + href, e1);
+ }
+ }
+
+ if (url != null)
+ res.add(url.toString());
+ }
+ }
+
+ NodeList childNodes = node.getChildNodes();
+
+ for (int i = 0; i < childNodes.getLength(); i++)
+ findReferences(childNodes.item(i), res, baseUrl, scanCtx);
+ }
+
+ /**
+ * @param url Base URL string format.
+ * @return File name extracted from {@code url} string format.
+ */
+ private String fileName(String url) {
+ assert url != null;
+
+ return url.substring(url.lastIndexOf('/') + 1);
+ }
+
+ /**
+ * Gets file URI for the given file name. It extends any given name with {@code URI}.
+ *
+ * @param name File name.
+ * @param scanCtx Scan context.
+ * @return URI for the given file name.
+ */
+ private String getFileUri(String name, UriDeploymentScannerContext scanCtx) {
+ assert name != null;
+
+ String fileUri = scanCtx.getUri().toString();
+
+ fileUri = fileUri.length() > 0 && fileUri.charAt(fileUri.length() - 1) == '/' ? fileUri + name :
+ fileUri + '/' + name;
+
+ return fileUri;
+ }
+ }
+
+ /**
+ * Verifier always return successful result for any host.
+ */
+ private static class DeploymentHostnameVerifier implements HostnameVerifier {
+ /** {@inheritDoc} */
+ @Override public boolean verify(String hostname, SSLSession ses) {
+ // Remote host trusted by default.
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java
index 21e5eb5..32fdadc 100644
--- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java
+++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java
@@ -44,9 +44,6 @@ public class GridUriDeploymentMultiScannersErrorThrottlingTest extends GridUriDe
public List<String> getUriList() {
List<String> uriList = new ArrayList<>();
- uriList.add("ftp://anonymous:111111;freq=5000@unknown.host:21/pub/gg-test");
- uriList.add("ftp://anonymous:111111;freq=5000@localhost:21/pub/gg-test");
-
uriList.add("http://freq=5000@localhost/tasks");
uriList.add("http://freq=5000@unknownhost.host/tasks");
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java
index af54e06..1beee49 100644
--- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java
+++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java
@@ -42,8 +42,7 @@ public class GridUriDeploymentMultiScannersSelfTest extends GridUriDeploymentAbs
public List<String> getUriList() {
List<String> uriList = new ArrayList<>();
- // Fake URIs.
- uriList.add(GridTestProperties.getProperty("deploy.uri.ftp"));
+ // Fake URI.
uriList.add(GridTestProperties.getProperty("deploy.uri.http"));
// One real URI.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java
index 2d6d681..0ac7c98 100644
--- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java
+++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java
@@ -70,7 +70,7 @@ public class GridFileDeploymentUndeploySelfTest extends GridSpiAbstractTest<UriD
assert newGarFile.exists();
- Thread.sleep(UriDeploymentSpi.DFLT_DISK_SCAN_FREQUENCY + 3000);
+ Thread.sleep(UriDeploymentFileScanner.DFLT_SCAN_FREQ + 3000);
assert getSpi().findResource("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3") != null :
"Failed to find resource for added GAR file.";
@@ -82,7 +82,7 @@ public class GridFileDeploymentUndeploySelfTest extends GridSpiAbstractTest<UriD
assert !newGarFile.exists();
- Thread.sleep(UriDeploymentSpi.DFLT_DISK_SCAN_FREQUENCY + 3000);
+ Thread.sleep(UriDeploymentFileScanner.DFLT_SCAN_FREQ + 3000);
assert getSpi().findResource("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3") == null;
assert getSpi().findResource("GridUriDeploymentTestWithNameTask3") == null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java
deleted file mode 100644
index 5b73889..0000000
--- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.deployment.uri.scanners.ftp;
-
-import org.apache.ignite.spi.deployment.uri.*;
-import org.apache.ignite.testframework.config.*;
-import org.apache.ignite.testframework.junits.spi.*;
-
-import java.util.*;
-
-/**
- * Test for FTP deployment.
- */
-@GridSpiTest(spi = UriDeploymentSpi.class, group = "Deployment SPI")
-public class GridFtpDeploymentTest extends GridUriDeploymentAbstractSelfTest {
- /**
- * @throws Exception If failed.
- */
- public void testDeployment() throws Exception {
- checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3");
- checkTask("GridUriDeploymentTestWithNameTask3");
- }
-
- /**
- * @return List of deployment sources.
- */
- @GridSpiTestConfig
- public List<String> getUriList() {
- return Collections.singletonList(GridTestProperties.getProperty("deploy.uri.ftp"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
----------------------------------------------------------------------
diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
deleted file mode 100644
index 50fd545..0000000
--- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains internal tests or test related classes and interfaces.
- */
-package org.apache.ignite.spi.deployment.uri.scanners.ftp;
\ No newline at end of file