You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/10/13 15:52:47 UTC
ambari git commit: AMBARI-13345. Views: HDFS API utility needs to
work with WASB (Gaurav Nagar via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 0da6327ac -> 45ce79631
AMBARI-13345. Views: HDFS API utility needs to work with WASB (Gaurav Nagar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/45ce7963
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/45ce7963
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/45ce7963
Branch: refs/heads/trunk
Commit: 45ce79631b95066664e3a8a01661f1daa9739dd2
Parents: 0da6327
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Oct 13 06:52:33 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Oct 13 06:52:33 2015 -0700
----------------------------------------------------------------------
.../view/hive/resources/files/FileService.java | 3 ++
contrib/views/utils/pom.xml | 5 +++
.../view/utils/hdfs/ConfigurationBuilder.java | 45 ++++++++++++++++++--
3 files changed, 50 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ce7963/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
index 6718093..1b8d7cc 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
@@ -75,6 +75,9 @@ public class FileService extends BaseService {
@Path("{filePath:.*}")
@Produces(MediaType.APPLICATION_JSON)
public Response getFilePage(@PathParam("filePath") String filePath, @QueryParam("page") Long page) throws IOException, InterruptedException {
+ if (!filePath.startsWith("/") && !filePath.startsWith(".")) {
+ filePath = "/" + filePath; // some servers strip double slashes in URL
+ }
LOG.debug("Reading file " + filePath);
try {
FileResource file = new FileResource();
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ce7963/contrib/views/utils/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/utils/pom.xml b/contrib/views/utils/pom.xml
index d9c3e4e..39ddc93 100644
--- a/contrib/views/utils/pom.xml
+++ b/contrib/views/utils/pom.xml
@@ -133,6 +133,11 @@
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
</dependencies>
<properties>
http://git-wip-us.apache.org/repos/asf/ambari/blob/45ce7963/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
index d992c2e..a739c84 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
@@ -23,6 +23,8 @@ import org.apache.ambari.view.utils.ambari.AmbariApi;
import org.apache.ambari.view.utils.ambari.NoClusterAssociatedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.fs.azure.Wasb;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.slf4j.Logger;
@@ -66,7 +68,7 @@ public class ConfigurationBuilder {
private AmbariApi ambariApi = null;
private AuthConfigurationBuilder authParamsBuilder;
private Map<String, String> authParams;
-
+ private URI defaultFsUri;
/**
* Constructor of ConfigurationBuilder based on ViewContext
* @param context ViewContext
@@ -87,12 +89,14 @@ public class ConfigurationBuilder {
LOG.info("HA HDFS cluster found.");
} else {
- if (!hasPort(defaultFS)) {
+ if (defaultFS.startsWith("hdfs://") && !hasPort(defaultFS)) {
defaultFS = addPortIfMissing(defaultFS);
}
}
- } catch (URISyntaxException e) {
+ defaultFsUri = new URI(defaultFS);
+
+ } catch (URISyntaxException e) {
throw new HdfsApiException("HDFS060 Invalid " + DEFAULT_FS_INSTANCE_PROPERTY +
"='" + defaultFS + "' URI", e);
}
@@ -121,6 +125,21 @@ public class ConfigurationBuilder {
return value;
}
+ private void copyPropertyIfExists(String type, String key) {
+ String value;
+ try {
+ value = ambariApi.getCluster().getConfigurationValue(type, key);
+ if (value != null) {
+ conf.set(key, value);
+ LOG.debug("set " + key + " = " + value);
+ } else {
+ LOG.debug("No such property " + type + "/" + key);
+ }
+ } catch (NoClusterAssociatedException e) {
+ LOG.debug("No such property " + type + "/" + key);
+ }
+ }
+
private void copyHAProperties(String defaultFS) throws URISyntaxException, HdfsApiException {
URI uri = new URI(defaultFS);
String nameservice = uri.getHost();
@@ -152,6 +171,7 @@ public class ConfigurationBuilder {
private String copyClusterProperty(String propertyName, String instancePropertyName) {
String value = getProperty(HDFS_SITE, propertyName, instancePropertyName);
conf.set(propertyName, value);
+ LOG.debug("set " + propertyName + " = " + value);
return value;
}
@@ -208,10 +228,29 @@ public class ConfigurationBuilder {
conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName());
conf.set("fs.file.impl", LocalFileSystem.class.getName());
+ configureWASB();
+
return conf;
}
/**
+ * Fill Azure Blob Storage properties if wasb:// scheme configured
+ */
+ public void configureWASB() {
+ LOG.debug("defaultFsUri.getScheme() == " + defaultFsUri.getScheme());
+ if (defaultFsUri.getScheme().equals("wasb")) {
+ conf.set("fs.AbstractFileSystem.wasb.impl", Wasb.class.getName());
+ conf.set("fs.wasb.impl", NativeAzureFileSystem.class.getName());
+
+ String account = defaultFsUri.getHost();
+ LOG.debug("WASB account == " + account);
+ copyPropertyIfExists(CORE_SITE, "fs.azure.account.key." + account);
+ copyPropertyIfExists(CORE_SITE, "fs.azure.account.keyprovider." + account);
+ copyPropertyIfExists(CORE_SITE, "fs.azure.shellkeyprovider.script");
+ }
+ }
+
+ /**
* Builds the authentication configuration
* @return map of HDFS auth params for view
* @throws HdfsApiException