You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/08/30 16:39:33 UTC
[04/52] [abbrv] oodt git commit: Altered configuration publishing
mechanism to use spring configuration
Altered configuration publishing mechanism to use spring configuration
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/fadf149c
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/fadf149c
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/fadf149c
Branch: refs/heads/master
Commit: fadf149cb77dd943af9e81c0f127ce74f203c478
Parents: 4b88f71
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Sat Jun 17 21:09:58 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Sat Jun 17 21:09:58 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 15 +++
.../oodt/config/ConfigurationManager.java | 19 +--
.../config/ConfigurationManagerFactory.java | 6 +-
.../java/org/apache/oodt/config/Constants.java | 3 +
.../DistributedConfigurationManager.java | 68 +---------
.../DistributedConfigurationPublisher.java | 135 ++++++++++---------
.../oodt/config/distributed/ZNodePaths.java | 80 +++++++++++
.../apache/oodt/config/distributed/ZPaths.java | 67 ---------
.../StandaloneConfigurationManager.java | 27 +---
.../apache/oodt/config/utils/CuratorUtils.java | 46 +++++++
.../DistributedConfigurationPublisherTest.java | 55 +++++---
.../resources/distributed-config-publisher.xml | 56 ++++++++
.../cas/filemgr/system/XmlRpcFileManager.java | 13 +-
13 files changed, 329 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/pom.xml
----------------------------------------------------------------------
diff --git a/config/pom.xml b/config/pom.xml
index 0ab4457..891f520 100644
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -28,6 +28,21 @@
</dependency>
<dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
index e5dc82f..1911889 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
@@ -17,11 +17,7 @@
package org.apache.oodt.config;
-import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
/**
* The abstract class to define functionalities of the configuration managers.
@@ -31,23 +27,10 @@ import java.util.List;
public abstract class ConfigurationManager {
protected String component;
- protected List<String> propertiesFiles;
- public ConfigurationManager(String component, List<String> propertiesFiles) {
+ public ConfigurationManager(String component) {
this.component = component;
- this.propertiesFiles = propertiesFiles != null ? propertiesFiles : new ArrayList<String>();
}
- /**
- * Retrieves a given property from the underlying configuration storage. For example, If we want to get the
- * value of the property org.foo.bar, we have to call this method with <pre>org.foo.bar</pre> as the parameter.
- *
- * @param key Name of the property to be retrieved.
- * @return Value of the requested property | null
- */
- public abstract String getProperty(String key);
-
public abstract void loadProperties() throws IOException;
-
- public abstract File getPropertiesFile(String filePath) throws FileNotFoundException;
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
index 0fd995d..4d01e35 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
@@ -43,11 +43,15 @@ public class ConfigurationManagerFactory {
* the distributed version of the configuration manager will be determined by the value of the property
* <pre>org.apache.oodt.config.zookeeper == true</pre>
*
+ * @param component Name of the OODT component, to which the created configuration manager instance will be providing
+ * configuration support
+ * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone configuration
+ * management
* @return ConfigurationManager instance to used by the corresponding component.
*/
public static ConfigurationManager getConfigurationManager(String component, List<String> propertiesFiles) {
if (System.getProperty(ENABLE_DISTRIBUTED_CONFIGURATION) != null) {
- return new DistributedConfigurationManager(component, propertiesFiles);
+ return new DistributedConfigurationManager(component);
}
return new StandaloneConfigurationManager(component, propertiesFiles);
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 6f74f97..033101b 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -70,5 +70,8 @@ public class Constants {
/** Where properties files are stored inside each component */
public static final String PROPERTIES_PATH_NAME = "properties";
+
+ /** Where other configuration files will be stored */
+ public static final String CONFIGURATION_PATH_NAME = "configuration";
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index f6502bb..9ab0e32 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -18,21 +18,14 @@
package org.apache.oodt.config.distributed;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.api.ACLProvider;
-import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.Constants.Properties;
import org.apache.oodt.config.utils.CuratorUtils;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
-import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
@@ -52,8 +45,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private String connectString;
private CuratorFramework client;
- public DistributedConfigurationManager(String component, List<String> propertiesFiles) {
- super(component, propertiesFiles);
+ public DistributedConfigurationManager(String component) {
+ super(component);
if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
@@ -82,45 +75,12 @@ public class DistributedConfigurationManager extends ConfigurationManager {
* {@link Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
*/
private void startZookeeper() {
- int connectionTimeoutMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "15"));
- int sessionTimeoutMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "60"));
- int retryInitialWaitMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "1000"));
- int maxRetryCount = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "3"));
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000"));
-
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .connectString(connectString)
- .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
- .connectionTimeoutMs(connectionTimeoutMs)
- .sessionTimeoutMs(sessionTimeoutMs);
-
- /*
- * If authorization information is available, those will be added to the client. NOTE: These auth info are
- * for access control, therefore no authentication will happen when the client is being started. These
- * info will only be required whenever a client is accessing an already create ZNode. For another client of
- * another node to make use of a ZNode created by this node, it should also provide the same auth info.
- */
- if (System.getProperty(Properties.ZK_USERNAME) != null && System.getProperty(Properties.ZK_PASSWORD) != null) {
- String authenticationString = System.getProperty(Properties.ZK_USERNAME) + ":" + System.getProperty(Properties.ZK_PASSWORD);
- builder.authorization("digest", authenticationString.getBytes())
- .aclProvider(new ACLProvider() {
- public List<ACL> getDefaultAcl() {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
-
- public List<ACL> getAclForPath(String path) {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
- });
- }
-
- client = builder.build();
- logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}",
- connectString, sessionTimeoutMs, connectionTimeoutMs);
+ client = CuratorUtils.getCuratorFrameworkClient(connectString, logger);
client.start();
logger.info("Curator framework start operation invoked");
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000"));
try {
logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
@@ -136,27 +96,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
}
@Override
- public String getProperty(String key) {
- // Todo Implement using curator
- return null;
- }
-
- @Override
public void loadProperties() {
// todo Implement the logic with Curator
}
-
- public File getPropertiesFile(String filePath) {
- return null;
- }
-
- public File getConfigurationFile(String filePath) {
- return null;
- }
-
- public void publishConfiguration() {
- for (String propertyFile : propertiesFiles) {
-
- }
- }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
index 38aea56..8970641 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -19,24 +19,19 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.api.ACLProvider;
-import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.utils.CuratorUtils;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -44,7 +39,8 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
/**
- * The class to publish configuration to Zookeeper
+ * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per
+ * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that.
*
* @author Imesha Sudasingha
*/
@@ -53,13 +49,15 @@ public class DistributedConfigurationPublisher {
private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class);
private Map<String, String> propertiesFiles;
+ private Map<String, String> configFiles;
private String connectString;
private CuratorFramework client;
- private ZPaths zPaths;
+ private ZNodePaths zNodePaths;
+ private String componentName;
- public DistributedConfigurationPublisher(String component, Map<String, String> propertiesFiles) {
- this.propertiesFiles = propertiesFiles;
- this.zPaths = new ZPaths(component);
+ public DistributedConfigurationPublisher(String componentName) {
+ this.componentName = componentName;
+ this.zNodePaths = new ZNodePaths(componentName);
if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
@@ -88,45 +86,12 @@ public class DistributedConfigurationPublisher {
* {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
*/
private void startZookeeper() {
- int connectionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "15000"));
- int sessionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "60000"));
- int retryInitialWaitMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "1000"));
- int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3"));
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
-
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .connectString(connectString)
- .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
- .connectionTimeoutMs(connectionTimeoutMs)
- .sessionTimeoutMs(sessionTimeoutMs);
-
- /*
- * If authorization information is available, those will be added to the client. NOTE: These auth info are
- * for access control, therefore no authentication will happen when the client is being started. These
- * info will only be required whenever a client is accessing an already create ZNode. For another client of
- * another node to make use of a ZNode created by this node, it should also provide the same auth info.
- */
- if (System.getProperty(Constants.Properties.ZK_USERNAME) != null && System.getProperty(Constants.Properties.ZK_PASSWORD) != null) {
- String authenticationString = System.getProperty(Constants.Properties.ZK_USERNAME) + ":" + System.getProperty(Constants.Properties.ZK_PASSWORD);
- builder.authorization("digest", authenticationString.getBytes())
- .aclProvider(new ACLProvider() {
- public List<ACL> getDefaultAcl() {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
-
- public List<ACL> getAclForPath(String path) {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
- });
- }
-
- client = builder.build();
- logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}",
- connectString, sessionTimeoutMs, connectionTimeoutMs);
+ client = CuratorUtils.getCuratorFrameworkClient(connectString, logger);
client.start();
logger.info("Curator framework start operation invoked");
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
try {
logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
@@ -142,8 +107,18 @@ public class DistributedConfigurationPublisher {
}
public void publishConfiguration() throws Exception {
- for (Map.Entry<String, String> entry : propertiesFiles.entrySet()) {
- logger.info("Publishing configuration {} - {}", entry.getKey(), entry.getValue());
+ logger.debug("Publishing properties files : {}", propertiesFiles);
+ publishConfiguration(propertiesFiles, true);
+ logger.info("Properties files published successfully");
+
+ logger.debug("Publishing config files : {}", configFiles);
+ publishConfiguration(configFiles, false);
+ logger.info("Config files published successfully");
+ }
+
+ private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
+ for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
+ logger.debug("Publishing configuration {} - {}", entry.getKey(), entry.getValue());
URL resource = Thread.currentThread().getContextClassLoader().getResource(entry.getKey());
String content;
@@ -154,17 +129,17 @@ public class DistributedConfigurationPublisher {
continue;
}
- String zNodePath = zPaths.getPropertiesZNodePath(entry.getValue());
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(entry.getValue()) : zNodePaths.getConfigurationZNodePath(entry.getValue());
if (client.checkExists().forPath(zNodePath) != null) {
byte[] bytes = client.getData().forPath(zNodePath);
String existingData = new String(bytes);
if (content.equals(existingData)) {
- logger.info("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue());
+ logger.warn("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue());
continue;
} else {
Stat stat = client.setData().forPath(zNodePath, content.getBytes());
if (stat != null) {
- logger.info("Published {} to {}", entry.getKey(), entry.getValue());
+ logger.info("Published configuration file {} to {}", entry.getKey(), entry.getValue());
}
}
} else {
@@ -173,28 +148,56 @@ public class DistributedConfigurationPublisher {
}
}
- public static void main(String[] args) {
- if (args.length < 2) {
- logger.warn("Component name and at least one properties file needs to be given configuration publishing");
- throw new IllegalArgumentException("Requires at least two arguments, component and one properties file");
- }
+ public Map<String, String> getPropertiesFiles() {
+ return propertiesFiles;
+ }
- logger.debug("Starting publishing configuration. Component : {}", args[0]);
+ public void setPropertiesFiles(Map<String, String> propertiesFiles) {
+ this.propertiesFiles = propertiesFiles;
+ }
+
+ public Map<String, String> getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(Map<String, String> configFiles) {
+ this.configFiles = configFiles;
+ }
+
+ public ZNodePaths getZNodePaths() {
+ return zNodePaths;
+ }
- String[] fileMapping = Arrays.copyOfRange(args, 1, args.length);
- Map<String, String> propertiesFiles = new HashMap<>();
- for (String entry : fileMapping) {
- String[] strings = entry.split("=");
- propertiesFiles.put(strings[0], strings[1]);
+ public static void main(String[] args) {
+ if (args.length < 1) {
+ logger.error("Spring configuration file needs to be given as an argument");
+ return;
}
- DistributedConfigurationPublisher distributedConfigurationPublisher = new DistributedConfigurationPublisher(args[0], propertiesFiles);
+ logger.info("Starting publishing configuration. Spring conf : {}", args[0]);
+
try {
- distributedConfigurationPublisher.publishConfiguration();
+ ApplicationContext applicationContext = new ClassPathXmlApplicationContext(args[0]);
+ Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
+ for (Object bean : distributedConfigurationPublisher.values()) {
+ DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+
+ logger.debug("Publishing configuration for : {}", publisher.getComponentName());
+ publisher.publishConfiguration();
+ logger.info("Published configuration for : {}", publisher.getComponentName());
+ }
+ } catch (BeansException e) {
+ logger.error("Error occurred when obtaining configuration publisher beans", e);
+ return;
} catch (Exception e) {
logger.error("Error occurred when publishing configuration to zookeeper", e);
+ return;
}
logger.info("Published configuration successfully");
}
+
+ public String getComponentName() {
+ return componentName;
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
new file mode 100644
index 0000000..f390838
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config.distributed;
+
+import static org.apache.oodt.config.Constants.ZPaths.*;
+
+/**
+ * Class responsible for handling all the Zookeeper ZNode paths related to configuration
+ *
+ * @author Imesha Sudasingha
+ */
+public class ZNodePaths {
+
+ /** ZNode for distinct components. /components/${component} */
+ private String componentZNodePath;
+ private String componentZNodeRoot;
+
+ /** ZNode path for properties files. /components/${component}/properties */
+ private String propertiesZNodePath;
+ private String propertiesZNodeRoot;
+
+ /** ZNode path for other configuration files. /components/${component}/configuration */
+ private String configurationZNodePath;
+ private String configurationZNodeRoot;
+
+ /**
+ * Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre> given.
+ *
+ * @param componentName Name of the OODT component
+ */
+ public ZNodePaths(String componentName) {
+ if (componentName == null) {
+ throw new IllegalArgumentException("Component name cannot be null");
+ }
+
+ componentZNodePath = SEPARATOR + COMPONENTS_PATH_NAME + SEPARATOR + componentName;
+ componentZNodeRoot = componentZNodePath + SEPARATOR;
+
+ propertiesZNodePath = componentZNodeRoot + PROPERTIES_PATH_NAME;
+ propertiesZNodeRoot = propertiesZNodePath + SEPARATOR;
+
+ configurationZNodePath = componentZNodeRoot + CONFIGURATION_PATH_NAME;
+ configurationZNodeRoot = configurationZNodePath + SEPARATOR;
+ }
+
+ public String getComponentZNodePath() {
+ return componentZNodePath;
+ }
+
+ public String getPropertiesZNodePath() {
+ return propertiesZNodePath;
+ }
+
+ public String getConfigurationZNodePath() {
+ return configurationZNodePath;
+ }
+
+ public String getPropertiesZNodePath(String subPath) {
+ return propertiesZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath);
+ }
+
+ public String getConfigurationZNodePath(String subPath) {
+ return configurationZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java
deleted file mode 100644
index 0e1d10c..0000000
--- a/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.config.distributed;
-
-import java.util.List;
-
-import static org.apache.oodt.config.Constants.ZPaths.*;
-
-/**
- * Class responsible for handling all the Zookeeper ZNode paths related to configuration
- *
- * @author Imesha Sudasingha
- */
-public class ZPaths {
-
- /** ZNode for distinct components. /components/${component} */
- public String componentZNodePath;
- public String componentZNodeRoot;
-
- /** ZNode path for properties files. /components/${component}/properties */
- public String propertiesZNodePath;
- public String propertiesZNodeRoot;
-
- /**
- * Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre> given.
- *
- * @param componentName Name of the OODT component
- */
- public ZPaths(String componentName) {
- if (componentName == null) {
- throw new IllegalArgumentException("Component name cannot be null");
- }
-
- componentZNodePath = SEPARATOR + COMPONENTS_PATH_NAME + SEPARATOR + componentName;
- componentZNodeRoot = componentZNodePath + SEPARATOR;
-
- propertiesZNodePath = componentZNodeRoot + PROPERTIES_PATH_NAME;
- propertiesZNodeRoot = propertiesZNodePath + SEPARATOR;
- }
-
- public String getComponentZNodePath() {
- return componentZNodePath;
- }
-
- public String getPropertiesZNodePath() {
- return propertiesZNodePath;
- }
-
- public String getPropertiesZNodePath(String subPath) {
- return propertiesZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath);
- }
-}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
index 23cf0a2..6ecb1b7 100644
--- a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java
@@ -23,8 +23,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -34,35 +34,22 @@ import java.util.List;
*/
public class StandaloneConfigurationManager extends ConfigurationManager {
- /** Logger instance for logging */
private static final Logger logger = LoggerFactory.getLogger(StandaloneConfigurationManager.class);
- public StandaloneConfigurationManager(String component, List<String> propertiesFiles) {
- super(component, propertiesFiles);
- }
+ private List<String> propertiesFiles;
- /** {@inheritDoc} */
- @Override
- public String getProperty(String key) {
- return System.getProperty(key);
+ public StandaloneConfigurationManager(String component, List<String> propertiesFiles) {
+ super(component);
+ this.propertiesFiles = propertiesFiles != null ? propertiesFiles : new ArrayList<String>();
}
/** {@inheritDoc} */
@Override
public void loadProperties() throws IOException {
for (String file : propertiesFiles) {
+ logger.debug("Loading properties from file : {}", file);
System.getProperties().load(new FileInputStream(new File(file)));
+ logger.debug("Properties loaded from file : {}", file);
}
}
-
- /** {@inheritDoc} */
- @Override
- public File getPropertiesFile(String filePath) throws FileNotFoundException {
- File file = new File(filePath);
- if (!propertiesFiles.contains(filePath) || !file.exists()) {
- throw new FileNotFoundException("Couldn't find properties file located at: " + filePath);
- }
-
- return file;
- }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java
index d064080..c73d48b 100644
--- a/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java
@@ -18,13 +18,20 @@
package org.apache.oodt.config.utils;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.oodt.config.Constants;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.IOException;
+import java.util.List;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
@@ -77,4 +84,43 @@ public class CuratorUtils {
return client.checkExists().forPath(path) != null ? new String(client.getData().forPath(path)) : null;
}
+ public static CuratorFramework getCuratorFrameworkClient(String connectString, Logger logger) {
+ int connectionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "15000"));
+ int sessionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "60000"));
+ int retryInitialWaitMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "1000"));
+ int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3"));
+
+ CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
+ .connectString(connectString)
+ .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
+ .connectionTimeoutMs(connectionTimeoutMs)
+ .sessionTimeoutMs(sessionTimeoutMs);
+
+ /*
+ * If authorization information is available, those will be added to the client. NOTE: These auth info are
+ * for access control, therefore no authentication will happen when the client is being started. These
+ * info will only be required whenever a client is accessing an already create ZNode. For another client of
+ * another node to make use of a ZNode created by this node, it should also provide the same auth info.
+ */
+ if (System.getProperty(Constants.Properties.ZK_USERNAME) != null && System.getProperty(Constants.Properties.ZK_PASSWORD) != null) {
+ String authenticationString = System.getProperty(Constants.Properties.ZK_USERNAME) + ":" + System.getProperty(Constants.Properties.ZK_PASSWORD);
+ builder.authorization("digest", authenticationString.getBytes())
+ .aclProvider(new ACLProvider() {
+ public List<ACL> getDefaultAcl() {
+ return ZooDefs.Ids.CREATOR_ALL_ACL;
+ }
+
+ public List<ACL> getAclForPath(String path) {
+ return ZooDefs.Ids.CREATOR_ALL_ACL;
+ }
+ });
+ }
+
+ CuratorFramework client = builder.build();
+ logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}",
+ connectString, sessionTimeoutMs, connectionTimeoutMs);
+
+ return client;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
index 665f785..2a3c903 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
@@ -26,17 +26,20 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.util.Arrays;
+import java.util.Map;
-import static org.apache.oodt.config.Constants.Components.FILE_MANAGER;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
public class DistributedConfigurationPublisherTest {
+ private static final String DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG = "distributed-config-publisher.xml";
+
private static TestingServer zookeeper;
private static CuratorFramework client;
@@ -56,27 +59,43 @@ public class DistributedConfigurationPublisherTest {
@Test
public void publishConfiguration() throws Exception {
- String args[] = new String[]{
- FILE_MANAGER,
- "filemgr.properties=/etc/filemgr.properties",
- "mime-types.xml=/etc/mime-types.xml"
- };
- DistributedConfigurationPublisher.main(args);
+ DistributedConfigurationPublisher.main(new String[]{DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG});
+
+ ApplicationContext applicationContext = new ClassPathXmlApplicationContext(DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG);
+ Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
+
+ for (Object bean : distributedConfigurationPublisher.values()) {
+ DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+ ZNodePaths zNodePaths = publisher.getZNodePaths();
+
+ // Checking for configuration files
+ for (Map.Entry<String, String> entry : publisher.getPropertiesFiles().entrySet()) {
+ String zNodePath = zNodePaths.getPropertiesZNodePath(entry.getValue());
+
+ Assert.assertNotNull(client.checkExists().forPath(zNodePath));
+
+ String storedContent = new String(client.getData().forPath(zNodePath));
+
+ URI file = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()).toURI();
+ String fileContent = FileUtils.readFileToString(new File(file));
+
+ Assert.assertEquals(fileContent, storedContent);
+ }
+
+ // Checking for configuration files
+ for (Map.Entry<String, String> entry : publisher.getConfigFiles().entrySet()) {
+ String zNodePath = zNodePaths.getConfigurationZNodePath(entry.getValue());
- String[] fileMappings = Arrays.copyOfRange(args, 1, args.length);
- ZPaths zPaths = new ZPaths(FILE_MANAGER);
- for (String mapping : fileMappings) {
- String[] parts = mapping.split("=");
- String zNodePath = zPaths.getPropertiesZNodePath(parts[1]);
+ Assert.assertNotNull(client.checkExists().forPath(zNodePath));
- Assert.assertNotNull(client.checkExists().forPath(zNodePath));
+ String storedContent = new String(client.getData().forPath(zNodePath));
- String storedContent = new String(client.getData().forPath(zNodePath));
+ URI file = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()).toURI();
+ String fileContent = FileUtils.readFileToString(new File(file));
- URI file = Thread.currentThread().getContextClassLoader().getResource(parts[0]).toURI();
- String fileContent = FileUtils.readFileToString(new File(file));
+ Assert.assertEquals(fileContent, storedContent);
+ }
- Assert.assertEquals(fileContent, storedContent);
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/test/resources/distributed-config-publisher.xml
----------------------------------------------------------------------
diff --git a/config/src/test/resources/distributed-config-publisher.xml b/config/src/test/resources/distributed-config-publisher.xml
new file mode 100644
index 0000000..d938b6d
--- /dev/null
+++ b/config/src/test/resources/distributed-config-publisher.xml
@@ -0,0 +1,56 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+
+ <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
+
+ <constructor-arg value="filemgr"/>
+
+ <property name="propertiesFiles">
+ <map>
+ <entry key="filemgr.properties" value="/etc/filemgr.properties"/>
+ </map>
+ </property>
+
+ <property name="configFiles">
+ <map>
+ <entry key="mime-types.xml" value="/etc/mime-types.xml"/>
+ </map>
+ </property>
+ </bean>
+
+ <bean id="resmgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
+
+ <constructor-arg value="resmgr"/>
+
+ <property name="propertiesFiles">
+ <map>
+ <entry key="filemgr.properties" value="/etc/resmgr.properties"/>
+ </map>
+ </property>
+
+ <property name="configFiles">
+ <map>
+ <entry key="mime-types.xml" value="/etc/mime-types.xml"/>
+ </map>
+ </property>
+ </bean>
+
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
index d43369a..9d7d740 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
@@ -125,18 +125,17 @@ public class XmlRpcFileManager {
webServer.addHandler("filemgr", this);
webServer.start();
+ List<String> propertiesFiles = new ArrayList<>();
+
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
- String configFile = System.getProperty("org.apache.oodt.cas.filemgr.properties");
- LOG.log(Level.INFO, "Loading File Manager Configuration Properties from: [" + configFile + "]");
-
- List<String> propertiesFiles = new ArrayList<String>();
+ String propertiesFile = System.getProperty("org.apache.oodt.cas.filemgr.properties");
+ LOG.log(Level.INFO, "Using File Manager Configuration Properties file : [" + propertiesFile + "]");
propertiesFiles.add(System.getProperty("org.apache.oodt.cas.filemgr.properties"));
- configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName, propertiesFiles);
- } else {
- configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName,null);
}
+ configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName, propertiesFiles);
+
this.loadConfiguration();
LOG.log(Level.INFO, "File Manager started by " + System.getProperty("user.name", "unknown"));
}