You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2017/04/03 10:05:52 UTC
[2/2] ambari git commit: AMBARI-20596. Cleanup temporary files needed
for downloading client configurations response (Attila Magyar via adoroszlai)
AMBARI-20596. Cleanup temporary files needed for downloading client configurations response (Attila Magyar via adoroszlai)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2d13f6a0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2d13f6a0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2d13f6a0
Branch: refs/heads/branch-2.5
Commit: 2d13f6a0591620058209aac175a8be3f349af362
Parents: f8497d2
Author: Attila Magyar <am...@hortonworks.com>
Authored: Mon Apr 3 12:05:16 2017 +0200
Committer: Attila Doroszlai <ad...@hortonworks.com>
Committed: Mon Apr 3 12:06:10 2017 +0200
----------------------------------------------------------------------
.../libraries/script/script.py | 8 +++--
.../internal/ClientConfigResourceProvider.java | 35 +++++++++++++++-----
.../ClientConfigResourceProviderTest.java | 11 +++---
.../stacks/2.0.6/HDFS/test_hdfs_client.py | 11 +++---
4 files changed, 44 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d13f6a0/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 6fb7b2e..9a5da04 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -847,6 +847,7 @@ class Script(object):
Directory(self.get_tmp_dir(), create_parents = True)
conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
+ os.chmod(conf_tmp_dir, 0700)
output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file'])
try:
@@ -854,22 +855,23 @@ class Script(object):
for filename, dict in file_dict.iteritems():
XmlConfig(filename,
conf_dir=conf_tmp_dir,
- mode=0644,
+ mode=0600,
**self.generate_configs_get_xml_file_content(filename, dict)
)
for file_dict in env_configs_list:
for filename,dicts in file_dict.iteritems():
File(os.path.join(conf_tmp_dir, filename),
- mode=0644,
+ mode=0600,
content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts)))
for file_dict in properties_configs_list:
for filename, dict in file_dict.iteritems():
PropertiesFile(os.path.join(conf_tmp_dir, filename),
- mode=0644,
+ mode=0600,
properties=self.generate_configs_get_xml_file_dict(filename, dict)
)
with closing(tarfile.open(output_filename, "w:gz")) as tar:
+ os.chmod(output_filename, 0600)
try:
tar.add(conf_tmp_dir, arcname=os.path.basename("."))
finally:
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d13f6a0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index b690e3a..446ca14 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -213,6 +213,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey());
String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey());
List<String> pythonCompressFilesCmds = new ArrayList<>();
+ List<File> commandFiles = new ArrayList<>();
for (ServiceComponentHostResponse response : componentMap.values()){
@@ -445,8 +446,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
jsonContent.put("clusterName", cluster.getClusterName());
jsonConfigurations = gson.toJson(jsonContent);
- File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json");
- File tmpDirectory = new File(jsonFileName.getParent());
+ File tmpDirectory = new File(TMP_PATH);
if (!tmpDirectory.exists()) {
try {
tmpDirectory.mkdirs();
@@ -456,22 +456,33 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
throw new SystemException("Failed to get temporary directory to store configurations", se);
}
}
+ File jsonFile = File.createTempFile(componentName, "-configuration.json", tmpDirectory);
+ try {
+ jsonFile.setWritable(true, true);
+ jsonFile.setReadable(true, true);
+ } catch (SecurityException e) {
+ throw new SystemException("Failed to set permission", e);
+ }
+
PrintWriter printWriter = null;
try {
- printWriter = new PrintWriter(jsonFileName.getAbsolutePath());
+ printWriter = new PrintWriter(jsonFile.getAbsolutePath());
printWriter.print(jsonConfigurations);
printWriter.close();
} catch (FileNotFoundException e) {
throw new SystemException("Failed to write configurations to json file ", e);
}
- String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " +
+ String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFile.getAbsolutePath() + " " +
packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH;
+ commandFiles.add(jsonFile);
pythonCompressFilesCmds.add(cmd);
} catch (AmbariException e) {
throw new SystemException("Controller error ", e);
+ } catch (IOException e) {
+ throw new SystemException("Controller error ", e);
}
}
@@ -484,11 +495,17 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize);
// put all threads that starts process to compress each component config files in the executor
- List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds);
-
- // wait for all threads to finish
- Integer timeout = configs.getExternalScriptTimeout();
- waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout);
+ try {
+ List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor, pythonCompressFilesCmds);
+
+ // wait for all threads to finish
+ Integer timeout = configs.getExternalScriptTimeout();
+ waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout);
+ } finally {
+ for (File each : commandFiles) {
+ each.delete();
+ }
+ }
if (StringUtils.isEmpty(requestComponentName)) {
TarUtils tarUtils;
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d13f6a0/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
index 84678e6..65d979b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.controller.internal;
import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
@@ -26,7 +27,6 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.powermock.api.mockito.PowerMockito.whenNew;
import java.io.ByteArrayInputStream;
@@ -360,6 +360,9 @@ public class ClientConfigResourceProviderTest {
expect(configHelper.createUserGroupsMap(stackId, cluster, desiredConfigMap)).andReturn(userGroupsMap).anyTimes();
PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(newFile).anyTimes();
+ PowerMock.mockStatic(File.class);
+ expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(newFile);
+
String commandLine = "ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs "+newFile +
" /tmp/stacks/S1/V1/PIG/package /var/lib/ambari-server/tmp/structured-out.json " +
"INFO /var/lib/ambari-server/tmp";
@@ -398,8 +401,7 @@ public class ClientConfigResourceProviderTest {
Set<Resource> resources = provider.getResources(request, predicate);
assertFalse(resources.isEmpty());
- String str = FileUtils.readFileToString(newFile);
- assertTrue(str.contains("\"user_groups\":\"{\\\"hdfsUser\\\":[\\\"hdfsGroup\\\"]}"));
+ assertFalse(newFile.exists());
// verify
verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition,
@@ -577,8 +579,9 @@ public class ClientConfigResourceProviderTest {
userSet.add("hdfs");
expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet);
PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes();
+ PowerMock.mockStatic(File.class);
+ expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(PowerMock.createNiceMock(File.class));
PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject());
- expect(mockFile.getParent()).andReturn("");
PowerMock.mockStatic(Runtime.class);
expect(mockFile.exists()).andReturn(true);
String commandLine = "ambari-python-wrap " + commonServicesPath + "/PIG/package/null generate_configs null " +
http://git-wip-us.apache.org/repos/asf/ambari/blob/2d13f6a0/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index b2636ab..85098fa 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -32,6 +32,7 @@ from resource_management.libraries.script.script import Script
@patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123'))
@patch.object(contextlib,"closing", new = MagicMock())
@patch("os.path.exists", new = MagicMock(return_value=True))
+@patch("os.chmod", new = MagicMock(return_value=True))
class Test(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
@@ -49,25 +50,25 @@ class Test(RMFTestCase):
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
conf_dir = '/tmp/123',
- mode=0644,
+ mode=0600,
configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site'],
configurations = self.getConfig()['configurations']['hdfs-site'],
)
self.assertResourceCalled('File', '/tmp/123/hadoop-env.sh',
- mode=0644,
+ mode=0600,
content = InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
)
self.assertResourceCalled('File', '/tmp/123/log4j.properties',
- mode=0644,
+ mode=0600,
content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+
self.getConfig()['configurations']['yarn-log4j']['content']),
)
self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
- mode=0644,
+ mode=0600,
properties = UnknownConfigurationMock(),
)
self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
- mode=0644,
+ mode=0600,
properties = UnknownConfigurationMock(),
)
self.assertResourceCalled('Directory', '/tmp/123',