You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ni...@apache.org on 2018/05/14 06:30:46 UTC
[ambari] branch trunk updated: AMBARI-23817 : Visualizing the
Encrypted zones and Erasure coded zones in HDFS (nitirajrathore) (#1243)
This is an automated email from the ASF dual-hosted git repository.
nitiraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1f6ebfe AMBARI-23817 : Visualizing the Encrypted zones and Erasure coded zones in HDFS (nitirajrathore) (#1243)
1f6ebfe is described below
commit 1f6ebfedf2ba167c00c44ba2d8fdf1c6e08717b5
Author: nitirajrathore <ni...@gmail.com>
AuthorDate: Mon May 14 12:00:43 2018 +0530
AMBARI-23817 : Visualizing the Encrypted zones and Erasure coded zones in HDFS (nitirajrathore) (#1243)
---
contrib/views/ambari-views-package/pom.xml | 2 +-
contrib/views/commons/pom.xml | 6 ++
contrib/views/files/pom.xml | 1 -
.../main/resources/ui/app/config/files-columns.js | 20 +++++-
.../main/resources/ui/app/helpers/shorten-text.js | 4 ++
.../files/src/main/resources/ui/app/models/file.js | 3 +
.../ui/app/templates/components/file-row.hbs | 10 +++
contrib/views/hive-next/pom.xml | 2 -
.../resources/uploads/parsers/ParseUtils.java | 4 +-
contrib/views/hive20/pom.xml | 2 -
.../resources/uploads/parsers/ParseUtils.java | 4 +-
contrib/views/hueambarimigration/pom.xml | 4 +-
contrib/views/jobs/pom.xml | 1 -
contrib/views/pig/pom.xml | 21 ++++--
.../org/apache/ambari/view/pig/test/JobTest.java | 26 ++++----
.../ambari/view/pig/test/ScriptTestUnmanaged.java | 3 +-
contrib/views/pom.xml | 66 ++++++++++++++++++-
contrib/views/storm/pom.xml | 6 +-
contrib/views/tez/pom.xml | 5 +-
contrib/views/utils/pom.xml | 11 +++-
.../org/apache/ambari/view/utils/hdfs/HdfsApi.java | 14 +++-
.../ambari/view/utils/hdfs/DummyFileStatus.java | 76 ++++++++++++++++++++++
.../view/utils/hdfs/DummyNonHdfsFileStatus.java} | 22 ++-----
.../apache/ambari/view/utils/hdfs/HdfsApiTest.java | 63 ++++++++++++++++++
24 files changed, 317 insertions(+), 59 deletions(-)
diff --git a/contrib/views/ambari-views-package/pom.xml b/contrib/views/ambari-views-package/pom.xml
index 16c9985..1ed662e 100644
--- a/contrib/views/ambari-views-package/pom.xml
+++ b/contrib/views/ambari-views-package/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>ambari-views-package</artifactId>
- <version>2.5.0.0.0</version>
+ <version>2.0.0.0-SNAPSHOT</version>
<name>Ambari Views Package</name>
<parent>
<groupId>org.apache.ambari.contrib.views</groupId>
diff --git a/contrib/views/commons/pom.xml b/contrib/views/commons/pom.xml
index 5018d46..e380122 100644
--- a/contrib/views/commons/pom.xml
+++ b/contrib/views/commons/pom.xml
@@ -58,6 +58,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
diff --git a/contrib/views/files/pom.xml b/contrib/views/files/pom.xml
index f7adbb5..acd7632 100644
--- a/contrib/views/files/pom.xml
+++ b/contrib/views/files/pom.xml
@@ -17,7 +17,6 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>files</artifactId>
<version>1.0.0.0-SNAPSHOT</version>
<name>Files</name>
diff --git a/contrib/views/files/src/main/resources/ui/app/config/files-columns.js b/contrib/views/files/src/main/resources/ui/app/config/files-columns.js
index 4dce394..d1caff7 100644
--- a/contrib/views/files/src/main/resources/ui/app/config/files-columns.js
+++ b/contrib/views/files/src/main/resources/ui/app/config/files-columns.js
@@ -26,7 +26,7 @@ var columnsConfig = [
isVisible: true,
sortable: true,
sortOrder: 0,
- columnClass: 'col-md-4 col-xs-4'
+ columnClass: 'col-md-3 col-xs-3'
},
{
title: 'Size',
@@ -50,7 +50,7 @@ var columnsConfig = [
isVisible: true,
sortable: true,
sortOrder: 0,
- columnClass: 'col-md-2 col-xs-2'
+ columnClass: 'col-md-1 col-xs-1'
},
{
title: 'Group',
@@ -66,7 +66,23 @@ var columnsConfig = [
isVisible: true,
sortable: false,
sortOrder: 0,
+ columnClass: 'col-md-1 col-xs-1'
+ },
+ {
+ title: 'Erasure Coding',
+ key: 'erasureCodingPolicyName',
+ isVisible: true,
+ sortable: false,
+ sortOrder: 0,
columnClass: 'col-md-2 col-xs-2'
+ },
+ {
+ title: 'Encrypted',
+ key: 'isEncrypted',
+ isVisible: true,
+ sortable: false,
+ sortOrder: 0,
+ columnClass: 'col-md-1 col-xs-1'
}
];
diff --git a/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js b/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js
index c50b5ca..f95ae85 100644
--- a/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js
+++ b/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js
@@ -20,6 +20,10 @@ import Ember from 'ember';
export function shortenText(params) {
let text = params[0];
+ if(!text){
+ return text;
+ }
+
let length = params[1];
if (text.length < length) {
return text;
diff --git a/contrib/views/files/src/main/resources/ui/app/models/file.js b/contrib/views/files/src/main/resources/ui/app/models/file.js
index a2ed1f3..eb26c0f 100644
--- a/contrib/views/files/src/main/resources/ui/app/models/file.js
+++ b/contrib/views/files/src/main/resources/ui/app/models/file.js
@@ -33,6 +33,9 @@ export default DS.Model.extend({
modificationTime : DS.attr('iso-date'),
blockSize : DS.attr('number'),
replication : DS.attr('number'),
+ isEncrypted : DS.attr('boolean'),
+ isErasureCoded : DS.attr('boolean'),
+ erasureCodingPolicyName : DS.attr('string'),
size : Ember.computed.alias('len'),
path: function() {
diff --git a/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs b/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
index 5198504..9b0ba12 100644
--- a/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
+++ b/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
@@ -43,4 +43,14 @@
<div class={{get-value-from-columns columnHeaders 'permission' 'columnClass'}}>
{{show-permission-string file}}
</div>
+ <div class={{get-value-from-columns columnHeaders 'erasureCodingPolicyName' 'columnClass'}}>
+ {{file.erasureCodingPolicyName}}
+ </div>
+ <div class={{get-value-from-columns columnHeaders 'isEncrypted' 'columnClass'}}>
+ {{#if file.isEncrypted}}
+ Yes
+ {{else}}
+ No
+ {{/if}}
+ </div>
</div>
diff --git a/contrib/views/hive-next/pom.xml b/contrib/views/hive-next/pom.xml
index 9a6c65d..60ddce3 100644
--- a/contrib/views/hive-next/pom.xml
+++ b/contrib/views/hive-next/pom.xml
@@ -17,7 +17,6 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>hive-jdbc</artifactId>
<version>2.0.0.0-SNAPSHOT</version>
<name>Hive Jdbc</name>
@@ -87,7 +86,6 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/parsers/ParseUtils.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/parsers/ParseUtils.java
index cafe148..34a1b8b 100644
--- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/parsers/ParseUtils.java
+++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/parsers/ParseUtils.java
@@ -18,7 +18,7 @@
package org.apache.ambari.view.hive2.resources.uploads.parsers;
-import org.apache.directory.api.util.Strings;
+import com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,7 +121,7 @@ public class ParseUtils {
return true;
String str = object.toString();
- if (Strings.isNotEmpty(str)) {
+ if (!Strings.isNullOrEmpty(str)) {
str = str.trim();
if (str.matches(HIVE_DATE_FORMAT_REGEX)) {
try {
diff --git a/contrib/views/hive20/pom.xml b/contrib/views/hive20/pom.xml
index 0da3c88..c851a40 100644
--- a/contrib/views/hive20/pom.xml
+++ b/contrib/views/hive20/pom.xml
@@ -17,7 +17,6 @@
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>hive20</artifactId>
<version>2.0.0.0-SNAPSHOT</version>
<name>Hive 2.0</name>
@@ -88,7 +87,6 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/parsers/ParseUtils.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/parsers/ParseUtils.java
index 27d03f2..1a22b6b 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/parsers/ParseUtils.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/parsers/ParseUtils.java
@@ -18,7 +18,7 @@
package org.apache.ambari.view.hive20.resources.uploads.parsers;
-import org.apache.directory.api.util.Strings;
+import com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,7 +121,7 @@ public class ParseUtils {
return true;
String str = object.toString();
- if (Strings.isNotEmpty(str)) {
+ if (!Strings.isNullOrEmpty(str)) {
str = str.trim();
if (str.matches(HIVE_DATE_FORMAT_REGEX)) {
try {
diff --git a/contrib/views/hueambarimigration/pom.xml b/contrib/views/hueambarimigration/pom.xml
index ec24679..72100e0 100644
--- a/contrib/views/hueambarimigration/pom.xml
+++ b/contrib/views/hueambarimigration/pom.xml
@@ -19,7 +19,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>hueambarimigration</artifactId>
-<version>1.0.0.0-SNAPSHOT</version>
+<version>2.0.0.0-SNAPSHOT</version>
<name>hueambarimigration</name>
<parent>
@@ -51,7 +51,7 @@
<dependency>
<groupId>org.apache.ambari</groupId>
<artifactId>ambari-views</artifactId>
- <version>[1.7.0.0,)</version>
+ <version>2.0.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
diff --git a/contrib/views/jobs/pom.xml b/contrib/views/jobs/pom.xml
index ea8a527..37f0cf7 100644
--- a/contrib/views/jobs/pom.xml
+++ b/contrib/views/jobs/pom.xml
@@ -199,7 +199,6 @@
<dependency>
<groupId>org.apache.ambari</groupId>
<artifactId>ambari-views</artifactId>
- <version>${ambari.version}</version>
</dependency>
<dependency>
diff --git a/contrib/views/pig/pom.xml b/contrib/views/pig/pom.xml
index 1430f69..1f5e2bf 100644
--- a/contrib/views/pig/pom.xml
+++ b/contrib/views/pig/pom.xml
@@ -17,9 +17,8 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>pig</artifactId>
- <version>1.0.0.0-SNAPSHOT</version>
+ <version>2.0.0.0-SNAPSHOT</version>
<name>Pig</name>
<parent>
@@ -61,6 +60,12 @@
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
@@ -75,7 +80,6 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -154,7 +158,16 @@
</properties>
<build>
<plugins>
- <plugin>
+ <plugin>
+ <!--disabled parallel execution of tests for pig-->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>0</forkCount>
+ </configuration>
+
+ </plugin>
+ <plugin>
<!-- Cleanup ui files -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java
index b46fc7e..dc1ae72 100644
--- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java
+++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java
@@ -108,7 +108,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -143,7 +143,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream).anyTimes();
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -184,7 +184,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -210,7 +210,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -245,7 +245,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -267,8 +267,8 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream baScriptStream = new ByteArrayOutputStream();
ByteArrayOutputStream baTempletonArgsStream = new ByteArrayOutputStream();
- FSDataOutputStream scriptStream = new FSDataOutputStream(baScriptStream);
- FSDataOutputStream templetonArgsStream = new FSDataOutputStream(baTempletonArgsStream);
+ FSDataOutputStream scriptStream = new FSDataOutputStream(baScriptStream, null);
+ FSDataOutputStream templetonArgsStream = new FSDataOutputStream(baTempletonArgsStream, null);
expect(hdfsApi.create(endsWith("script.pig"), eq(true))).andReturn(scriptStream);
expect(hdfsApi.create(endsWith("params"), eq(true))).andReturn(templetonArgsStream);
replay(hdfsApi);
@@ -294,7 +294,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -317,7 +317,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -339,7 +339,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -362,7 +362,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -397,7 +397,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
@@ -432,7 +432,7 @@ public class JobTest extends BasePigTest {
ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
- FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+ FSDataOutputStream stream = new FSDataOutputStream(do_stream, null);
expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
replay(hdfsApi);
UserLocalObjects.setHdfsApi(hdfsApi, context);
diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java
index 23f9fba..9cfd60c 100644
--- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java
+++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java
@@ -22,6 +22,7 @@ import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
import org.apache.ambari.view.pig.BasePigTest;
import org.apache.ambari.view.pig.resources.scripts.ScriptService;
+import org.apache.ambari.view.pig.utils.ServiceFormattedException;
import org.apache.ambari.view.utils.UserLocal;
import org.apache.ambari.view.utils.hdfs.HdfsApi;
import org.apache.ambari.view.utils.hdfs.HdfsApiException;
@@ -76,7 +77,7 @@ public class ScriptTestUnmanaged extends BasePigTest {
replay(handler, context);
scriptService = getService(ScriptService.class, handler, context);
- thrown.expect(HdfsApiException.class);
+ thrown.expect(ServiceFormattedException.class);
doCreateScript("Test", null);
}
}
diff --git a/contrib/views/pom.xml b/contrib/views/pom.xml
index ef8ed27..3d4efbc 100644
--- a/contrib/views/pom.xml
+++ b/contrib/views/pom.xml
@@ -31,7 +31,8 @@
<properties>
<ambari.version>2.0.0.0-SNAPSHOT</ambari.version>
<ambari.dir>${project.parent.parent.basedir}</ambari.dir>
- <hadoop.version>2.7.3</hadoop.version>
+ <jetty.version>9.3.19.v20170502</jetty.version>
+ <hadoop.version>3.0.0.3.0.0.0-1301</hadoop.version>
<views.jars.dir>views-jars</views.jars.dir>
<views.jars.dir.rel>../target/${views.jars.dir}</views.jars.dir.rel>
</properties>
@@ -165,6 +166,69 @@
</profiles>
<dependencyManagement>
<dependencies>
+ <!--We had to include the jetty's and other's version specific to the one used in hadoop and override the ambari managed version because the tests were failing. -->
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>3.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util-ajax</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.websocket</groupId>
+ <artifactId>websocket-servlet</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.websocket</groupId>
+ <artifactId>websocket-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
diff --git a/contrib/views/storm/pom.xml b/contrib/views/storm/pom.xml
index 4a1ff68..6f1ae10 100644
--- a/contrib/views/storm/pom.xml
+++ b/contrib/views/storm/pom.xml
@@ -26,8 +26,7 @@
<ui.directory>${basedir}/src/main/resources/ui</ui.directory>
</properties>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
- <artifactId>storm-view</artifactId>
+ <artifactId>storm-view</artifactId>
<name>Storm_Monitoring</name>
<version>0.1.0.0</version>
<description>Storm Monitoring View</description>
@@ -180,13 +179,12 @@
<dependency>
<groupId>org.apache.ambari</groupId>
<artifactId>ambari-views</artifactId>
- <version>${ambari.version}</version>
+ <version>2.0.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
<scope>provided</scope>
</dependency>
diff --git a/contrib/views/tez/pom.xml b/contrib/views/tez/pom.xml
index fa1a773..0c058a0 100644
--- a/contrib/views/tez/pom.xml
+++ b/contrib/views/tez/pom.xml
@@ -27,8 +27,7 @@
<tez.version>0.7.0</tez.version>
</properties>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
- <artifactId>tez-view</artifactId>
+ <artifactId>tez-view</artifactId>
<name>Tez</name>
<version>${tez.view.version}</version>
<description>Tez View</description>
@@ -222,7 +221,7 @@
<dependency>
<groupId>org.apache.ambari</groupId>
<artifactId>ambari-views</artifactId>
- <version>${ambari.version}</version>
+ <version>2.0.0.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/contrib/views/utils/pom.xml b/contrib/views/utils/pom.xml
index 82b87c8..63eac8c 100644
--- a/contrib/views/utils/pom.xml
+++ b/contrib/views/utils/pom.xml
@@ -17,7 +17,6 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>ambari-views-utils</artifactId>
<version>2.0.0.0-SNAPSHOT</version>
<name>Ambari View Utils</name>
@@ -76,6 +75,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
@@ -117,7 +121,6 @@
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
-
<dependency>
<groupId>org.glassfish.jersey.test-framework</groupId>
<artifactId>jersey-test-framework-core</artifactId>
@@ -171,6 +174,10 @@
<artifactId>commons-validator</artifactId>
<version>1.4.0</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ </dependency>
</dependencies>
<properties>
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
index 812cd54..1f8b584 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.security.UserGroupInformation;
import org.json.simple.JSONArray;
import org.slf4j.Logger;
@@ -52,6 +53,9 @@ public class HdfsApi {
private final static Logger LOG =
LoggerFactory.getLogger(HdfsApi.class);
+ public static String KeyIsErasureCoded = "isErasureCoded";
+ public static String KeyIsEncrypted = "isEncrypted";
+ public static String KeyErasureCodingPolicyName = "erasureCodingPolicyName";
private final Configuration conf;
private Map<String, String> authParams;
@@ -554,7 +558,6 @@ public class HdfsApi {
Map<String, Object> json = new LinkedHashMap<String, Object>();
json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath())
.toString());
- json.put("replication", status.getReplication());
json.put("isDirectory", status.isDirectory());
json.put("len", status.getLen());
json.put("owner", status.getOwner());
@@ -567,6 +570,15 @@ public class HdfsApi {
json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi));
json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi));
json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi));
+ json.put(KeyIsErasureCoded, status.isErasureCoded());
+ json.put(KeyIsEncrypted, status.isEncrypted());
+
+ if( status instanceof HdfsFileStatus){
+ HdfsFileStatus hdfsFileStatus = (HdfsFileStatus) status;
+ if(null != hdfsFileStatus.getErasureCodingPolicy()) {
+ json.put(KeyErasureCodingPolicyName, hdfsFileStatus.getErasureCodingPolicy().getName());
+ }
+ }
return json;
}
diff --git a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyFileStatus.java b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyFileStatus.java
new file mode 100644
index 0000000..eb419d6
--- /dev/null
+++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyFileStatus.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.view.utils.hdfs;
+
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+
+public abstract class DummyFileStatus extends FileStatus implements HdfsFileStatus {
+ @Override
+ public long getFileId() {
+ return 0;
+ }
+
+ @Override
+ public FileEncryptionInfo getFileEncryptionInfo() {
+ return null;
+ }
+
+ @Override
+ public byte[] getLocalNameInBytes() {
+ return new byte[0];
+ }
+
+ @Override
+ public byte[] getSymlinkInBytes() {
+ return new byte[0];
+ }
+
+ @Override
+ public int getChildrenNum() {
+ return 0;
+ }
+
+ @Override
+ public ErasureCodingPolicy getErasureCodingPolicy() {
+ return null;
+ }
+
+ @Override
+ public byte getStoragePolicy() {
+ return 0;
+ }
+
+ @Override
+ public void setPermission(FsPermission fsPermission) {
+
+ }
+
+ @Override
+ public void setOwner(String s) {
+
+ }
+
+ @Override
+ public void setGroup(String s) {
+
+ }
+}
diff --git a/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyNonHdfsFileStatus.java
similarity index 69%
copy from contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js
copy to contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyNonHdfsFileStatus.java
index c50b5ca..1989c47 100644
--- a/contrib/views/files/src/main/resources/ui/app/helpers/shorten-text.js
+++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/DummyNonHdfsFileStatus.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,17 +16,9 @@
* limitations under the License.
*/
-import Ember from 'ember';
+package org.apache.ambari.view.utils.hdfs;
-export function shortenText(params) {
- let text = params[0];
- let length = params[1];
- if (text.length < length) {
- return text;
- } else {
- return text.substring(0, length - 3) + '...';
- }
+import org.apache.hadoop.fs.FileStatus;
-}
-
-export default Ember.Helper.helper(shortenText);
+public abstract class DummyNonHdfsFileStatus extends FileStatus {
+}
\ No newline at end of file
diff --git a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
index e7a6752..928d596 100644
--- a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
+++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
@@ -23,7 +23,10 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -32,6 +35,11 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Map;
+
+import static org.apache.ambari.view.utils.hdfs.HdfsApi.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class HdfsApiTest {
private FileSystem fs;
@@ -62,6 +70,61 @@ public class HdfsApiTest {
}
@Test
+ public void testWith_EC_And_Encryption(){
+// Have to mock DummyFileStatus, because we cannot rely on internal class of hdfs
+ DummyFileStatus fileStatus = mock(DummyFileStatus.class);
+ FsPermission fsPermission = new FsPermission((short)0777);
+ String ecPolicyName = "Some-EC-Policy";
+ ECSchema ecSchema = new ECSchema("someSchema", 1, 1);
+ ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy(ecPolicyName, ecSchema, 1024, (byte)0);
+ when(fileStatus.getPermission()).thenReturn(fsPermission);
+ when(fileStatus.getPath()).thenReturn(new Path("/test/path"));
+ when(fileStatus.getErasureCodingPolicy()).thenReturn(erasureCodingPolicy);
+ when(fileStatus.isErasureCoded()).thenReturn(true);
+ when(fileStatus.isEncrypted()).thenReturn(true);
+ Map<String, Object> json = hdfsApi.fileStatusToJSON(fileStatus);
+
+ Assert.assertEquals(Boolean.TRUE, json.get(KeyIsErasureCoded));
+ Assert.assertEquals(Boolean.TRUE, json.get(KeyIsEncrypted));
+ Assert.assertEquals(json.get(KeyErasureCodingPolicyName), ecPolicyName);
+ }
+
+ @Test
+ public void testWithout_EC_And_Encryption(){
+// Have to mock DummyFileStatus, because we cannot rely on internal class of hdfs
+ FsPermission fsPermission = new FsPermission((short)0777);
+
+ DummyFileStatus fileStatus = mock(DummyFileStatus.class);
+ when(fileStatus.getPermission()).thenReturn(fsPermission);
+
+ when(fileStatus.getPath()).thenReturn(new Path("/test/path"));
+ when(fileStatus.getErasureCodingPolicy()).thenReturn(null);
+ when(fileStatus.isErasureCoded()).thenReturn(false);
+ when(fileStatus.isEncrypted()).thenReturn(false);
+ Map<String, Object> json = hdfsApi.fileStatusToJSON(fileStatus);
+
+ Assert.assertEquals(Boolean.FALSE, json.get(KeyIsErasureCoded));
+ Assert.assertEquals(Boolean.FALSE, json.get(KeyIsEncrypted));
+ Assert.assertNull(json.get(KeyErasureCodingPolicyName));
+ }
+
+ @Test
+ public void testNonHdfsFileStatus(){
+// Have to mock DummyNonHdfsFileStatus, because we cannot rely on internal class of hdfs
+ DummyNonHdfsFileStatus fileStatus = mock(DummyNonHdfsFileStatus.class);
+ FsPermission fsPermission = new FsPermission((short)0777);
+ when(fileStatus.getPermission()).thenReturn(fsPermission);
+ when(fileStatus.getPath()).thenReturn(new Path("/test/path"));
+
+ when(fileStatus.isErasureCoded()).thenReturn(false);
+ when(fileStatus.isEncrypted()).thenReturn(false);
+ Map<String, Object> json = hdfsApi.fileStatusToJSON(fileStatus);
+
+ Assert.assertEquals(Boolean.FALSE, json.get(KeyIsErasureCoded));
+ Assert.assertEquals( Boolean.FALSE,json.get(KeyIsEncrypted));
+ Assert.assertNull(json.get(KeyErasureCodingPolicyName));
+ }
+ @Test
public void filterAndTruncateDirStatus() throws Exception {
{
// null fileStatuses
--
To stop receiving notification emails like this one, please contact
nitiraj@apache.org.