You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/05/11 21:32:49 UTC
[4/8] ambari git commit: Revert "AMBARI-11034. Implement HdfsResouces
and it's usage (aonishuk)"
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/default.json b/ambari-server/src/test/python/stacks/2.2/configs/default.json
index 8188928..c5a6ae7 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/default.json
@@ -185,18 +185,7 @@
"ignore_groupsusers_create": "false",
"smokeuser": "ambari-qa",
"kerberos_domain": "EXAMPLE.COM",
- "user_group": "hadoop",
- "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz",
- "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
- "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz",
- "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",
- "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/",
- "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
- "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz",
- "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/",
- "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz",
- "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",
- "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz"
+ "user_group": "hadoop"
},
"ranger-knox-plugin-properties": {
"POLICY_MGR_URL": "{{policymgr_mgr_url}}",
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.2/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/secured.json b/ambari-server/src/test/python/stacks/2.2/configs/secured.json
index e224ebc..5bd8814 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/secured.json
@@ -173,18 +173,7 @@
"user_group": "hadoop",
"smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
"smokeuser_principal_name": "ambari-qa@EXAMPLE.COM",
- "kinit_path_local": "/usr/bin",
- "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz",
- "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
- "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz",
- "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",
- "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/",
- "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
- "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz",
- "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/",
- "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz",
- "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",
- "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz"
+ "kinit_path_local": "/usr/bin"
},
"webhcat-site": {
"templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
index 665119f..0d943c4 100644
--- a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
@@ -33,42 +33,35 @@ class TestMahoutClient(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
+ self.assertResourceCalled('ExecuteHadoop', 'fs -rm -r -f /user/ambari-qa/mahoutsmokeoutput /user/ambari-qa/mahoutsmokeinput',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ try_sleep = 5,
+ kinit_path_local = '/usr/bin/kinit',
+ tries = 3,
+ user = 'ambari-qa',
+ bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ principal = UnknownConfigurationMock(),
+ )
+ self.assertResourceCalled('ExecuteHadoop', 'fs -mkdir /user/ambari-qa/mahoutsmokeinput',
+ try_sleep = 5,
+ tries = 3,
+ bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ user = 'ambari-qa',
+ conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
self.assertResourceCalled('File', '/tmp/sample-mahout-test.txt',
- content = 'Test text which will be converted to sequence file.',
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- user = 'hdfs',
- owner = 'ambari-qa',
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- type = 'directory',
- action = ['create_on_execute'],
- )
- self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput/sample-mahout-test.txt',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- source = '/tmp/sample-mahout-test.txt',
- user = 'hdfs',
- owner = 'ambari-qa',
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- type = 'file',
- action = ['create_on_execute'],
- )
- self.assertResourceCalled('HdfsResource', None,
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- user = 'hdfs',
- action = ['execute'],
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- )
+ content = 'Test text which will be converted to sequence file.',
+ mode = 0755,
+ )
+ self.assertResourceCalled('ExecuteHadoop', 'fs -put /tmp/sample-mahout-test.txt /user/ambari-qa/mahoutsmokeinput/',
+ try_sleep = 5,
+ tries = 3,
+ bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ user = 'ambari-qa',
+ conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
self.assertResourceCalled('Execute', 'mahout seqdirectory --input /user/ambari-qa/mahoutsmokeinput/'
'sample-mahout-test.txt --output /user/ambari-qa/mahoutsmokeoutput/ '
'--charset utf-8',
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/dependency-reduced-pom.xml b/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
deleted file mode 100644
index c252f2e..0000000
--- a/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.ambari</groupId>
- <artifactId>fast-hdfs-resource</artifactId>
- <name>fast-hdfs-resource</name>
- <version>0.0.1-SNAPSHOT</version>
- <url>http://maven.apache.org</url>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer>
- <mainClass>org.apache.ambari.fast_hdfs_resource.Runner</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <repositories>
- <repository>
- <id>hdp.internal</id>
- <url>http://repo1.maven.org/maven2</url>
- </repository>
- </repositories>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-</project>
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/pom.xml b/contrib/fast-hdfs-resource/pom.xml
deleted file mode 100644
index 6a92841..0000000
--- a/contrib/fast-hdfs-resource/pom.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.ambari</groupId>
- <artifactId>fast-hdfs-resource</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- <packaging>jar</packaging>
-
- <name>fast-hdfs-resource</name>
- <url>http://maven.apache.org</url>
- <repositories>
- <repository>
- <id>hdp.internal</id>
- <url>http://repo1.maven.org/maven2</url>
- </repository>
- </repositories>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-tools</artifactId>
- <version>1.2.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.2.1</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.2.2</version>
- </dependency>
- </dependencies>
-
-
- <!-- Create executable jar with the application entry point -->
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.ambari.fast_hdfs_resource.Runner
- </mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/resources/example.json
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/resources/example.json b/contrib/fast-hdfs-resource/resources/example.json
deleted file mode 100644
index 605deb9..0000000
--- a/contrib/fast-hdfs-resource/resources/example.json
+++ /dev/null
@@ -1,57 +0,0 @@
-[
-{
- "target":"/tmp/some999",
- "type":"directory",
- "action":"delete"
-},
-{
- "target":"/tmp/some999/more/dirs/for/recursive/tests",
- "type":"directory",
- "action":"create"
-},
-{
- "target":"/tmp/some999/more/dirs/for/recursive/tests/file_empty.txt",
- "type":"file",
- "action":"create"
-},
-{
- "target":"/tmp/some999",
- "type":"directory",
- "action":"create",
- "owner":"oozie"
-},
-{
- "target":"/tmp/some999",
- "type":"directory",
- "action":"create",
- "group":"hive"
-},
-{
- "target":"/tmp/some999",
- "type":"directory",
- "action":"create",
- "mode":"777"
-},
-{
- "target":"/tmp/some999/more/dirs",
- "type":"directory",
- "action":"create",
- "owner":"yarn",
- "group":"mapred",
- "recursiveChown":true,
- "mode":"757",
- "recursiveChmod":true
-},
-{
- "source":"/tmp/my.txt",
- "target":"/tmp/some999/my_file.txt",
- "type":"file",
- "action":"create"
-},
-{
- "source":"/tmp/a",
- "target":"/tmp/some999/a_dir",
- "type":"directory",
- "action":"create"
-}
-]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/resources/test_perfomance.sh
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/resources/test_perfomance.sh b/contrib/fast-hdfs-resource/resources/test_perfomance.sh
deleted file mode 100644
index 40339e3..0000000
--- a/contrib/fast-hdfs-resource/resources/test_perfomance.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Delete 2 dirs
-sudo -u hdfs hadoop fs -rm -r /tmp/some999
-sudo -u hdfs hadoop fs -rm -r /tmp/some888
-# Create
-sudo -u hdfs hadoop fs -mkdir -p /tmp/some999/more/dirs/for/recursive/tests
-# Create + permissions + owner
-sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/more/dirs/for/recursive/tests
-sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/more/dirs/for/recursive/tests
-sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/more/dirs/for/recursive/tests
-# Empty dirs with permissions/owners to last dir"
-sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
-sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
-sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
-# Empty dirs with permissions/owners to last file
-sudo -u hdfs hadoop fs -touchz /tmp/some888/file.txt
-sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/file.txt
-sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/file.txt
-# Empty dirs with permissions/owners to last file
-sudo -u hdfs hadoop fs -touchz /tmp/some888/and_more/and_dirs/file2.txt
-sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/file2.txt
-sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/file2.txt
-# Recursive permissions
-sudo -u hdfs hadoop fs -chmod -R 700 /tmp/some888
-sudo -u hdfs hadoop fs -chown -R hive:hive /tmp/some999
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
deleted file mode 100644
index 21750e1..0000000
--- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.fast_hdfs_resource;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.File;
-import java.lang.System;
-import java.util.ArrayList;
-import java.util.HashSet;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.FileSystem;
-
-/**
- * Used to: 1) copy files/directories from localFS to hadoopFs 2) create empty
- * files/directories in hadoopFs
- */
-public class Resource {
- private String source;
- private String target;
- private String type;
- private String action;
- private String owner;
- private String group;
- private String mode;
- private boolean recursiveChown;
- private boolean recursiveChmod;
- private boolean changePermissionforParents;
-
- public String getSource() {
- return source;
- }
-
- public void setSource(String source) {
- this.source = source;
- }
-
- public String getTarget() {
- return target;
- }
-
- public void setTarget(String target) {
- this.target = target;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getAction() {
- return action;
- }
-
- public void setAction(String action) {
- this.action = action;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getGroup() {
- return group;
- }
-
- public void setGroup(String group) {
- this.group = group;
- }
-
- public String getMode() {
- return mode;
- }
-
- public void setMode(String mode) {
- this.mode = mode;
- }
-
- public boolean isRecursiveChown() {
- return recursiveChown;
- }
-
- public void setRecursiveChown(boolean recursiveChown) {
- this.recursiveChown = recursiveChown;
- }
-
- public boolean isRecursiveChmod() {
- return recursiveChmod;
- }
-
- public void setRecursiveChmod(boolean recursiveChmod) {
- this.recursiveChmod = recursiveChmod;
- }
-
- public boolean isChangePermissionOnParents() {
- return changePermissionforParents;
- }
-
- public void setChangePermissionOnParents(boolean changePermissionforParents) {
- this.changePermissionforParents = changePermissionforParents;
- }
-
-
-
-
- @Override
- public String toString() {
- return "Resource [source=" + source + ", target=" + target + ", type="
- + type + ", action=" + action + ", owner=" + owner + ", group=" + group
- + ", mode=" + mode + ", recursiveChown=" + recursiveChown
- + ", recursiveChmod=" + recursiveChmod
- + ", changePermissionforParents=" + changePermissionforParents + "]";
- }
-
- /*
- * Check if parameters are correctly set
- */
- public static void checkResourceParameters(Resource resource,
- FileSystem dfs) throws IllegalArgumentException, IOException {
-
- ArrayList<String> actionsAvailable = new ArrayList<String>();
- actionsAvailable.add("create");
- actionsAvailable.add("delete");
- ArrayList<String> typesAvailable = new ArrayList<String>();
- typesAvailable.add("file");
- typesAvailable.add("directory");
-
- if (resource.getTarget() == null)
- throw new IllegalArgumentException(
- "Path to resource in HadoopFs must be filled.");
-
- if (resource.getAction() == null
- || !actionsAvailable.contains(resource.getAction()))
- throw new IllegalArgumentException("Action is not supported.");
-
- if (resource.getType() == null
- || !typesAvailable.contains(resource.getType()))
- throw new IllegalArgumentException("Type is not supported.");
-
- // Check consistency for ("type":"file" == file in hadoop)
- if (dfs.isFile(new Path(resource.getTarget()))
- && !"file".equals(resource.getType()))
- throw new IllegalArgumentException(
- "Cannot create a directory " + resource.getTarget() +
- " because file is present on the given path.");
- // Check consistency for ("type":"directory" == directory in hadoop)
- else if (dfs.isDirectory(new Path(resource.getTarget()))
- && !"directory".equals(resource.getType()))
- throw new IllegalArgumentException(
- "Cannot create a file " + resource.getTarget() +
- " because directory is present on the given path.");
-
- if(resource.getSource() != null) {
- File source = new File(resource.getSource());
- if(source.isFile()
- && !"file".equals(resource.getType()))
- throw new IllegalArgumentException(
- "Cannot create a directory " + resource.getTarget() +
- " because source " + resource.getSource() + "is a file");
- else if(source.isDirectory()
- && !"directory".equals(resource.getType()))
- throw new IllegalArgumentException(
- "Cannot create a file " + resource.getTarget() +
- " because source " + resource.getSource() + "is a directory");
- }
- }
-
- /*
- * Create/copy resource - {type}
- */
- public static void createResource(Resource resource,
- FileSystem dfs, Path pathHadoop) throws IOException {
-
- boolean isCreate = (resource.getSource() == null) ? true : false;
-
- if (isCreate && resource.getType().equals("directory")) {
- dfs.mkdirs(pathHadoop); // empty dir(s)
- } else if (isCreate && resource.getType().equals("file")) {
- dfs.createNewFile(pathHadoop); // empty file
- } else {
- dfs.copyFromLocalFile(new Path(resource.getSource()), pathHadoop);// copy
- }
- }
-
- /*
- * Set permissions on resource - {mode}
- */
- public static void setMode(Resource resource,
- FileSystem dfs, Path pathHadoop) throws IOException {
-
- if (resource.getMode() != null) {
- FsPermission permission = new FsPermission(resource.getMode());
- dfs.setPermission(pathHadoop, permission);
-
- // Recursive
-
- // Get the list of sub-directories and files
- HashSet<String> resultSet = new HashSet<String>();
-
- if (resource.isRecursiveChmod())
- resource.fillDirectoryList(dfs, resource.getTarget(), resultSet);
-
- if(resource.isChangePermissionOnParents())
- resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet);
-
- for (String path : resultSet) {
- dfs.setPermission(new Path(path), permission);
- }
-
- }
- }
-
- /*
- * Set owner on resource - {owner}
- */
- public static void setOwner(Resource resource, FileSystem dfs,
- Path pathHadoop) throws IOException {
-
- if (!(resource.getOwner() == null && resource.getGroup() == null)) {
- dfs.setOwner(pathHadoop, resource.getOwner(), resource.getGroup());
-
- // Get the list of sub-directories and files
- HashSet<String> resultSet = new HashSet<String>();
- if (resource.isRecursiveChown())
- resource.fillDirectoryList(dfs, resource.getTarget(), resultSet);
- if(resource.isChangePermissionOnParents())
- resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet);
-
- for (String path : resultSet) {
- dfs.setOwner(new Path(path), resource.getOwner(), resource.getGroup());
- }
- }
- }
-
- public void fillInParentDirectories(FileSystem dfs, String path, HashSet<String> resultSet) throws IOException {
- Path filePath = new Path(path);
-
- while(true) {
- filePath = filePath.getParent();
-
- // if(filePath.isRoot()) {
- if(filePath.getParent() == null) {
- break;
- }
- resultSet.add(filePath.toString());
- }
- }
-
- /*
- * List all files and sub-directories recursively
- */
- public void fillDirectoryList(FileSystem dfs, String path,
- HashSet<String> resultSet) throws IOException {
-
- FileStatus[] fileStatus = dfs.listStatus(new Path(path));
- if (fileStatus != null) {
- // Go through all resources in directory
- for (FileStatus fs : fileStatus) {
- String pathToResource = path + "/" + fs.getPath().getName();
-
- resultSet.add(pathToResource);
-
- if (fs.isDir()) {
- // recursive
- fillDirectoryList(dfs, pathToResource, resultSet);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e833066e/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
deleted file mode 100644
index 291a2d9..0000000
--- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.fast_hdfs_resource;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-
-import com.google.gson.Gson;
-
-public class Runner {
- public static void main(String[] args)
- throws IOException, URISyntaxException {
- // 1 - Check arguments
- if (args.length != 1) {
- System.err.println("Incorrect number of arguments. Please provide:\n"
- + "1) Path to json file\n"
- + "Exiting...");
- System.exit(1);
- }
-
- // 2 - Check if json-file exists
- final String jsonFilePath = args[0];
- File file = new File(jsonFilePath);
-
- if (!file.isFile()) {
- System.err
- .println("File " + jsonFilePath + " doesn't exist.\nExiting...");
- System.exit(1);
- }
-
- Gson gson = new Gson();
- Resource[] resources = null;
- FileSystem dfs = null;
-
- try {
- Configuration conf = new Configuration();
- dfs = FileSystem.get(conf);
-
- // 3 - Load data from JSON
- resources = (Resource[]) gson.fromJson(new FileReader(jsonFilePath),
- Resource[].class);
-
- // 4 - Connect to HDFS
- System.out.println("Using filesystem uri: " + FileSystem.getDefaultUri(conf).toString());
- dfs.initialize(FileSystem.getDefaultUri(conf), conf);
-
- for (Resource resource : resources) {
- System.out.println("Creating: " + resource);
-
- Resource.checkResourceParameters(resource, dfs);
-
- Path pathHadoop = new Path(resource.getTarget());
- if (resource.getAction().equals("create")) {
- // 5 - Create
- Resource.createResource(resource, dfs, pathHadoop);
- Resource.setMode(resource, dfs, pathHadoop);
- Resource.setOwner(resource, dfs, pathHadoop);
- } else if (resource.getAction().equals("delete")) {
- // 6 - Delete
- dfs.delete(pathHadoop, true);
- }
- }
-
- } finally {
- dfs.close();
- }
-
- System.out.println("All resources created.");
- }
-
-}