You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/09/02 15:35:57 UTC
[37/50] [abbrv] oodt git commit: Clear configuration files option for
configuration manager
Clear configuration files option for configuration manager
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/72722e03
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/72722e03
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/72722e03
Branch: refs/heads/development
Commit: 72722e0339c522bfdeac3e13278e8682e9cf13df
Parents: e32f2bb
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Sun Aug 6 12:59:21 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Sun Aug 6 12:59:21 2017 +0530
----------------------------------------------------------------------
.../oodt/config/ConfigurationManager.java | 18 +++++
.../DistributedConfigurationManager.java | 72 ++++++++++++++++----
.../StandaloneConfigurationManager.java | 12 ++++
.../DistributedConfigurationManagerTest.java | 29 ++------
4 files changed, 97 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/72722e03/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
index 56fea0d..2554604 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
@@ -17,6 +17,8 @@
package org.apache.oodt.config;
+import java.util.List;
+
/**
* The abstract class to define functions of the configuration managers.
*
@@ -32,7 +34,23 @@ public abstract class ConfigurationManager {
public abstract void loadConfiguration() throws Exception;
+ /**
+ * Clears loaded configuration. Invocation of this method will remove the downloaded configuration files to be
+ * deleted in the distributed configuration management scenario. Any child class that is extending this class should
+ * implement this operation on their own.
+ */
+ public abstract void clearConfiguration();
+
public Component getComponent() {
return component;
}
+
+ /**
+ * Returns a list of file paths which are the locations of the files stored locally corresponding to configuration.
+ * In distributed configuration management scenario, this stands for the files downloaded and stored in local file
+ * system.
+ *
+ * @return list of locally stored files
+ */
+ public abstract List<String> getSavedFiles();
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/72722e03/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index 122a78e..6b6ef21 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -32,6 +32,7 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -55,6 +56,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private Component component;
private ZNodePaths zNodePaths;
+ private List<String> savedFiles = new ArrayList<>();
+
public DistributedConfigurationManager(Component component) {
super(component);
this.component = component;
@@ -78,20 +81,17 @@ public class DistributedConfigurationManager extends ConfigurationManager {
connectString = System.getProperty(Constants.Properties.ZK_CONNECT_STRING);
logger.info("Using zookeeper connect string : {}", connectString);
-
startZookeeper();
}
/**
- * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of
- * {@link Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
+ * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of {@link
+ * Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
*/
private void startZookeeper() {
client = CuratorUtils.newCuratorFrameworkClient(connectString, logger);
-
client.start();
logger.info("Curator framework start operation invoked");
-
int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000"));
try {
logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
@@ -141,10 +141,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
logger.info("Properties loaded from ZNode at : {}", propertiesFileZNodePath);
String localFilePath = zNodePaths.getLocalPropertiesFilePath(propertiesFileZNodePath);
- localFilePath = FilePathUtils.fixForComponentHome(component, localFilePath);
- logger.debug("Storing configuration in file: {}", localFilePath);
- FileUtils.writeByteArrayToFile(new File(localFilePath), bytes);
- logger.info("Properties file from ZNode at {} saved to {}", propertiesFileZNodePath, localFilePath);
+ saveFile(localFilePath, bytes);
}
}
@@ -168,16 +165,67 @@ public class DistributedConfigurationManager extends ConfigurationManager {
byte[] bytes = client.getData().forPath(configFileZNodePath);
String localFilePath = zNodePaths.getLocalConfigFilePath(configFileZNodePath);
- localFilePath = FilePathUtils.fixForComponentHome(component, localFilePath);
- FileUtils.writeByteArrayToFile(new File(localFilePath), bytes);
- logger.info("Config file from ZNode at {} saved to {}", configFileZNodePath, localFilePath);
+ saveFile(localFilePath, bytes);
+ }
+ }
+
+ private void saveFile(String path, byte[] data) throws IOException {
+ String localFilePath = FilePathUtils.fixForComponentHome(component, path);
+ File localFile = new File(localFilePath);
+ if (localFile.exists()) {
+ logger.warn("Deleting already existing file at {} before writing new content", localFilePath);
+ localFile.delete();
+ }
+
+ logger.debug("Storing configuration in file: {}", localFilePath);
+ FileUtils.writeByteArrayToFile(localFile, data);
+ logger.info("File from ZNode at {} saved to {}", path, localFilePath);
+ savedFiles.add(localFilePath);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearConfiguration() {
+ for (String path : savedFiles) {
+ logger.debug("Removing saved file {}", path);
+ File file = new File(path);
+ if (file.delete()) {
+ logger.debug("Deleted saved file {}", path);
+
+ int lastIndex = path.lastIndexOf(Constants.SEPARATOR);
+ String parentPath = path.substring(0, lastIndex == -1 ? 0 : lastIndex);
+ while (!parentPath.isEmpty()) {
+ // Deleting parent if empty
+ File parent = new File(parentPath);
+ File[] files = parent.listFiles();
+ if (files == null || files.length != 0) {
+ break;
+ }
+
+ if (!parent.delete()) {
+ break;
+ }
+ logger.debug("Deleted directory {} since it is empty", parentPath);
+ lastIndex = parentPath.lastIndexOf(Constants.SEPARATOR);
+ parentPath = path.substring(0, lastIndex == -1 ? 0 : lastIndex);
+ }
+ } else {
+ logger.warn("Unable to delete saved file {}", path);
+ }
}
+ savedFiles.clear();
}
public Component getComponent() {
return component;
}
+ /** {@inheritDoc} */
+ @Override
+ public List<String> getSavedFiles() {
+ return new ArrayList<>(savedFiles);
+ }
+
public ZNodePaths getzNodePaths() {
return zNodePaths;
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/72722e03/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
index cc3fe1a..94269bb 100644
--- a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
@@ -52,4 +52,16 @@ public class StandaloneConfigurationManager extends ConfigurationManager {
logger.debug("Properties loaded from file : {}", file);
}
}
+
+ /** {@inheritDoc} */
+ @Override
+ public void clearConfiguration() {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List<String> getSavedFiles() {
+ return new ArrayList<>();
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/72722e03/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
index 064d1c6..a4315a5 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
@@ -17,7 +17,6 @@
package org.apache.oodt.config.distributed;
-import org.apache.commons.io.FileUtils;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.distributed.cli.ConfigPublisher;
import org.apache.oodt.config.distributed.utils.FilePathUtils;
@@ -33,11 +32,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.Set;
import static org.apache.oodt.config.Constants.SEPARATOR;
import static org.junit.Assert.fail;
@@ -114,6 +111,13 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
File file = new File(fileName);
Assert.assertTrue(file.exists());
}
+
+ List<String> localFiles = configurationManager.getSavedFiles();
+ configurationManager.clearConfiguration();
+ for (String localFile : localFiles) {
+ File file = new File(localFile);
+ Assert.assertFalse(file.exists());
+ }
}
}
@@ -121,25 +125,6 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
public void tearDownTest() throws Exception {
for (DistributedConfigurationPublisher publisher : publishers) {
publisher.destroy();
-
- // deleting all locally created conf file directories
- Set<Map.Entry<String, String>> files = new HashSet<>(publisher.getConfigFiles().entrySet());
- files.addAll(publisher.getPropertiesFiles().entrySet());
-
- for (Map.Entry<String, String> entry : files) {
- String fileName = entry.getValue();
- fileName = fileName.startsWith(SEPARATOR) ? fileName.substring(1) : fileName;
-
- String prefixPath = System.getProperty(publisher.getComponent().getHome());
- if (prefixPath == null) {
- prefixPath = System.getenv(publisher.getComponent().getHome());
- }
- String confDir = prefixPath != null && !prefixPath.trim().isEmpty() ?
- prefixPath.trim() + SEPARATOR + fileName.split(SEPARATOR)[0] : fileName.split(SEPARATOR)[0];
-
- File dir = new File(confDir);
- FileUtils.deleteDirectory(dir);
- }
}
ConfigPublisher.main(new String[]{