You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/07/15 03:46:54 UTC
[12/35] oodt git commit: Added zip packaging for distributed
configuration publisher
Added zip packaging for distributed configuration publisher
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/27a28daa
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/27a28daa
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/27a28daa
Branch: refs/heads/feature/zookeeper-config
Commit: 27a28daa3d3c448b462bd9a33eb6ac4fd859b75a
Parents: 5fcf222
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Sun Jul 2 20:53:49 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Sun Jul 2 20:53:49 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 36 +-
config/src/main/assembly/assembly.xml | 56 +
.../java/org/apache/oodt/config/Constants.java | 7 +
.../DistributedConfigurationManager.java | 1 -
.../DistributedConfigurationPublisher.java | 208 -
.../config/distributed/cli/CmdLineOptions.java | 57 +
.../cli/DistributedConfigurationPublisher.java | 300 ++
.../config/distributed/utils/CuratorUtils.java | 1 +
.../src/main/resources/etc/config-publisher.xml | 36 +
config/src/main/resources/etc/log4j.xml | 34 +
.../main/resources/examples/filemgr.properties | 114 +
.../src/main/resources/examples/mime-types.xml | 4119 ++++++++++++++++++
.../DistributedConfigurationPublisherTest.java | 54 +-
.../oodt/config/distributed/TestServer.java | 45 +
.../resources/distributed-config-publisher.xml | 56 -
.../src/test/resources/etc/config-publisher.xml | 50 +
config/src/test/resources/filemgr.properties | 114 -
config/src/test/resources/log4j.xml | 34 -
config/src/test/resources/mime-types.xml | 4119 ------------------
core/pom.xml | 6 +
20 files changed, 4862 insertions(+), 4585 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/pom.xml
----------------------------------------------------------------------
diff --git a/config/pom.xml b/config/pom.xml
index 891f520..d75e032 100644
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -16,50 +16,68 @@
<description>OODT project for distributed configuration management support</description>
<dependencies>
-
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
-
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
-
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
</dependency>
-
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
-
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
-
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
-
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-2</version>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/config/src/main/assembly/assembly.xml b/config/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..2743cab
--- /dev/null
+++ b/config/src/main/assembly/assembly.xml
@@ -0,0 +1,56 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<assembly>
+ <id>dist</id>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
+ <includeSiteDirectory>false</includeSiteDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}</directory>
+ <outputDirectory>.</outputDirectory>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>CHANGES.txt</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes/>
+ <fileMode>755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <outputDirectory>.</outputDirectory>
+ <includes/>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpackOptions/>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 033101b..d6969fe 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -17,6 +17,8 @@
package org.apache.oodt.config;
+import java.io.File;
+
/**
* Constants to be used by the config package
*
@@ -27,12 +29,17 @@ public class Constants {
private Constants() {
}
+ public static final String SEPARATOR = File.separator;
+
/** Node name to be used when the configuration manager is the standalone version */
public static final String STANDALONE_NODE_NAME = "local";
/** Default environment name to be used */
public static final String DEFAULT_ENVIRONMENT = "default";
+ /** The XML file name in which the configuration to be published to zookeeper is defined */
+ public static final String CONFIG_PUBLISHER_XML = "etc" + SEPARATOR + "config-publisher.xml";
+
public static class Properties {
/** The system property to be set in order to enable distributed configuration management */
public static final String ENABLE_DISTRIBUTED_CONFIGURATION = "org.apache.oodt.config.distributed";
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index d16df7e..1a3f390 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -114,7 +114,6 @@ public class DistributedConfigurationManager extends ConfigurationManager {
logger.debug("Saving configuration files for : {}", componentName);
saveConfigFiles();
logger.info("Configuration files saved for : {}", componentName);
-
}
/**
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
deleted file mode 100644
index 37b57c9..0000000
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.config.distributed;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.oodt.config.Constants;
-import org.apache.oodt.config.distributed.utils.CuratorUtils;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
-import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
-
-/**
- * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per
- * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that.
- *
- * @author Imesha Sudasingha
- */
-public class DistributedConfigurationPublisher {
-
- private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class);
-
- private Map<String, String> propertiesFiles;
- private Map<String, String> configFiles;
- private String connectString;
- private CuratorFramework client;
- private ZNodePaths zNodePaths;
- private String componentName;
-
- public DistributedConfigurationPublisher(String componentName) {
- this.componentName = componentName;
- this.zNodePaths = new ZNodePaths(componentName);
-
- if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
- }
-
- if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
- try {
- CuratorUtils.loadZookeeperProperties();
- } catch (IOException e) {
- logger.error("Error occurred when loading properties from properties file");
- }
- }
-
- if (System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
- }
-
- connectString = System.getProperty(Constants.Properties.ZK_CONNECT_STRING);
- logger.info("Using zookeeper connect string : {}", connectString);
-
- startZookeeper();
- }
-
- /**
- * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of
- * {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
- */
- private void startZookeeper() {
- client = CuratorUtils.getCuratorFrameworkClient(connectString, logger);
-
- client.start();
- logger.info("Curator framework start operation invoked");
-
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
- try {
- logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
- client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
- } catch (InterruptedException ex) {
- logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString);
- }
-
- if (!client.getZookeeperClient().isConnected()) {
- throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString);
- }
-
- logger.info("CuratorFramework client started successfully");
- }
-
- public void publishConfiguration() throws Exception {
- logger.debug("Publishing properties files : {}", propertiesFiles);
- publishConfiguration(propertiesFiles, true);
- logger.info("Properties files published successfully");
-
- logger.debug("Publishing config files : {}", configFiles);
- publishConfiguration(configFiles, false);
- logger.info("Config files published successfully");
-
- // TODO: 6/17/17 Verify whether the given configuration are published correctly
- }
-
- private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
- for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
- logger.debug("Publishing configuration {} - {}", entry.getKey(), entry.getValue());
- URL resource = Thread.currentThread().getContextClassLoader().getResource(entry.getKey());
-
- String content;
- try {
- content = FileUtils.readFileToString(new File(resource.toURI()));
- } catch (IOException | URISyntaxException e) {
- logger.error("Unable to read file : {}", entry.getKey(), e);
- continue;
- }
-
- String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(entry.getValue()) : zNodePaths.getConfigurationZNodePath(entry.getValue());
- if (client.checkExists().forPath(zNodePath) != null) {
- byte[] bytes = client.getData().forPath(zNodePath);
- String existingData = new String(bytes);
- if (content.equals(existingData)) {
- logger.warn("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue());
- } else {
- Stat stat = client.setData().forPath(zNodePath, content.getBytes());
- if (stat != null) {
- logger.info("Published configuration file {} to {}", entry.getKey(), entry.getValue());
- }
- }
- } else {
- /*
- * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted
- * when no child node is present under them.
- */
- client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes());
- }
- }
- }
-
- public Map<String, String> getPropertiesFiles() {
- return propertiesFiles;
- }
-
- public void setPropertiesFiles(Map<String, String> propertiesFiles) {
- this.propertiesFiles = propertiesFiles;
- }
-
- public Map<String, String> getConfigFiles() {
- return configFiles;
- }
-
- public void setConfigFiles(Map<String, String> configFiles) {
- this.configFiles = configFiles;
- }
-
- public ZNodePaths getZNodePaths() {
- return zNodePaths;
- }
-
- public static void main(String[] args) {
- if (args.length < 1) {
- logger.error("Spring configuration file needs to be given as an argument");
- return;
- }
-
- logger.info("Starting publishing configuration. Spring conf : {}", args[0]);
-
- try {
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(args[0]);
- Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
- for (Object bean : distributedConfigurationPublisher.values()) {
- DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
-
- logger.debug("Publishing configuration for : {}", publisher.getComponentName());
- publisher.publishConfiguration();
- logger.info("Published configuration for : {}", publisher.getComponentName());
- }
- } catch (BeansException e) {
- logger.error("Error occurred when obtaining configuration publisher beans", e);
- return;
- } catch (Exception e) {
- logger.error("Error occurred when publishing configuration to zookeeper", e);
- return;
- }
-
- logger.info("Published configuration successfully");
- }
-
- public String getComponentName() {
- return componentName;
- }
-}
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java
new file mode 100644
index 0000000..afa6350
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config.distributed.cli;
+
+import org.kohsuke.args4j.Option;
+
+/**
+ * Bean class used to map CLI options and arguments provided by the user when publishing configuration to zookeeper
+ *
+ * @author Imesha Sudaingha
+ */
+public class CmdLineOptions {
+
+ @Option(name = "-connectString", usage = "Zookeeper connect string", required = true)
+ private String connectString;
+
+ @Option(name = "-publish", usage = "Publishes configuration specified in the spring config file to zookeeper. " +
+ "Any current similar config in zookeeper will be overwritten. If not specified, command will be assumed as a publish")
+ private boolean publish = true;
+
+ @Option(name = "-verify", usage = "Verifies the content in the local files and the published ones. Results will be printed.")
+ private boolean verify = true;
+
+ @Option(name = "-clear", usage = "Unpublish any configuration which has been published earlier using the same spring config file")
+ private boolean clear = false;
+
+ public String getConnectString() {
+ return connectString;
+ }
+
+ public boolean isPublish() {
+ return publish;
+ }
+
+ public boolean isVerify() {
+ return verify;
+ }
+
+ public boolean isClear() {
+ return clear;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
new file mode 100644
index 0000000..03cdfc7
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config.distributed.cli;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.oodt.config.Constants;
+import org.apache.oodt.config.distributed.ZNodePaths;
+import org.apache.oodt.config.distributed.utils.CuratorUtils;
+import org.apache.zookeeper.data.Stat;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
+import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
+
+/**
+ * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per
+ * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that.
+ *
+ * @author Imesha Sudasingha
+ */
+public class DistributedConfigurationPublisher {
+
+ private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class);
+
+ private Map<String, String> propertiesFiles;
+ private Map<String, String> configFiles;
+ private String connectString;
+ private CuratorFramework client;
+ private ZNodePaths zNodePaths;
+ private String componentName;
+
+ public DistributedConfigurationPublisher(String componentName) {
+ this.componentName = componentName;
+ this.zNodePaths = new ZNodePaths(componentName);
+
+ if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
+ }
+
+ if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
+ try {
+ CuratorUtils.loadZookeeperProperties();
+ } catch (IOException e) {
+ logger.error("Error occurred when loading properties from properties file");
+ }
+ }
+
+ if (System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
+ }
+
+ connectString = System.getProperty(ZK_CONNECT_STRING);
+ logger.info("Using zookeeper connect string : {}", connectString);
+
+ startZookeeper();
+ }
+
+ /**
+ * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of
+ * {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
+ */
+ private void startZookeeper() {
+ client = CuratorUtils.getCuratorFrameworkClient(connectString, logger);
+
+ client.start();
+ logger.info("Curator framework start operation invoked");
+
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
+ try {
+ logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
+ client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ex) {
+ logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString);
+ }
+
+ if (!client.getZookeeperClient().isConnected()) {
+ throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString);
+ }
+
+ logger.info("CuratorFramework client started successfully");
+ }
+
+ /**
+ * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being
+ * published, no further publishing attempts will be carried on. Error will be reported to user.
+ *
+ * @throws Exception Zookeeper errors
+ */
+ public void publishConfiguration() throws Exception {
+ logger.debug("Publishing properties files : {}", propertiesFiles);
+ publishConfiguration(propertiesFiles, true);
+ logger.info("Properties files published successfully");
+
+ logger.debug("Publishing config files : {}", configFiles);
+ publishConfiguration(configFiles, false);
+ logger.info("Config files published successfully");
+ }
+
+ /**
+ * Verified whether the actual content of the local files specified to be published are 100% similar to the ones that
+ * has been published and stored in zookeeper at the moment.
+ *
+ * @return true | if content are up to date and similar
+ */
+ public boolean verifyPublishedConfiguration() {
+ try {
+ return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false);
+ } catch (Exception e) {
+ logger.error("Error occurred when checking published config", e);
+ return false;
+ }
+ }
+
+ private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
+ for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
+ String filePath = entry.getKey();
+ String relativeZNodePath = entry.getValue();
+ logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath);
+
+ String content = getFileContent(filePath);
+
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ if (client.checkExists().forPath(zNodePath) != null) {
+ byte[] bytes = client.getData().forPath(zNodePath);
+ String existingData = new String(bytes);
+ if (content.equals(existingData)) {
+ logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath);
+ } else {
+ Stat stat = client.setData().forPath(zNodePath, content.getBytes());
+ if (stat != null) {
+ logger.info("Published configuration file {} to {}", filePath, relativeZNodePath);
+ } else {
+ logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath);
+ }
+ }
+ } else {
+ /*
+ * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted
+ * when no child node is present under them.
+ */
+ client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes());
+ logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath);
+ }
+ }
+ }
+
+ private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
+ boolean noError = true;
+ for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
+ String filePath = entry.getKey();
+ String relativeZNodePath = entry.getValue();
+ logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath);
+
+ String originalContent = getFileContent(filePath);
+
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ if (client.checkExists().forPath(zNodePath) == null) {
+ logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath);
+ noError = false;
+ continue;
+ }
+
+ String publishedContent = new String(client.getData().forPath(zNodePath));
+ if (!publishedContent.equals(originalContent)) {
+ logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath);
+ noError = false;
+ continue;
+ }
+
+ logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath);
+ }
+
+ return noError;
+ }
+
+ private String getFileContent(String file) {
+ String content;
+ try {
+ content = FileUtils.readFileToString(new File(file));
+ } catch (IOException e) {
+ logger.error("Unable to read file : {}", file, e);
+ throw new IllegalArgumentException("Unable to read content of the file : " + file);
+ }
+
+ return content;
+ }
+
+ public Map<String, String> getPropertiesFiles() {
+ return propertiesFiles;
+ }
+
+ public void setPropertiesFiles(Map<String, String> propertiesFiles) {
+ this.propertiesFiles = propertiesFiles;
+ }
+
+ public Map<String, String> getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(Map<String, String> configFiles) {
+ this.configFiles = configFiles;
+ }
+
+ public ZNodePaths getZNodePaths() {
+ return zNodePaths;
+ }
+
+ public static void main(String[] args) throws Exception {
+ CmdLineOptions cmdLineOptions = new CmdLineOptions();
+ CmdLineParser parser = new CmdLineParser(cmdLineOptions);
+
+ try {
+ parser.parseArgument(args);
+ } catch (CmdLineException e) {
+ System.err.println("There's an error in your command");
+ parser.printUsage(System.err);
+ return;
+ }
+
+ if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ System.err.println("Zookeeper connect string is not found");
+ parser.printUsage(System.err);
+ return;
+ } else {
+ System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString());
+ }
+
+ System.out.println("Starting configuration publishing");
+
+ try {
+ ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML);
+ Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
+
+ for (Object bean : distributedConfigurationPublisher.values()) {
+ DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+ System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponentName()));
+
+ if (cmdLineOptions.isPublish()) {
+ System.out.println(String.format("Publishing configuration for : %s", publisher.getComponentName()));
+ publisher.publishConfiguration();
+ System.out.println(String.format("Published configuration for : %s", publisher.getComponentName()));
+ System.out.printf("\n");
+ }
+
+ if (cmdLineOptions.isVerify()) {
+ System.out.println(String.format("Verifying configuration for : %s", publisher.getComponentName()));
+ publisher.verifyPublishedConfiguration();
+ System.out.println(String.format("Verified configuration for : %s", publisher.getComponentName()));
+ System.out.printf("\n");
+ }
+
+ if (cmdLineOptions.isClear()) {
+ System.out.println(String.format("Clearing configuration for : %s", publisher.getComponentName()));
+ // TODO: 7/2/17 Implement configuration removal
+ System.out.println(String.format("Cleared configuration for : %s", publisher.getComponentName()));
+ System.out.printf("\n");
+ }
+ }
+ } catch (BeansException e) {
+ logger.error("Error occurred when obtaining configuration publisher beans", e);
+ throw e;
+ } catch (Exception e) {
+ logger.error("Error occurred when publishing configuration to zookeeper", e);
+ throw e;
+ }
+
+ logger.info("Published configuration successfully");
+ }
+
+ public String getComponentName() {
+ return componentName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
index 6f9c97f..1b10ef0 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
@@ -100,6 +100,7 @@ public class CuratorUtils {
int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3"));
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
+ .namespace(NAMESPACE)
.connectString(connectString)
.retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
.connectionTimeoutMs(connectionTimeoutMs)
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/etc/config-publisher.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/etc/config-publisher.xml b/config/src/main/resources/etc/config-publisher.xml
new file mode 100644
index 0000000..cc1261e
--- /dev/null
+++ b/config/src/main/resources/etc/config-publisher.xml
@@ -0,0 +1,36 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+
+ <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher">
+ <constructor-arg value="filemgr"/>
+ <property name="propertiesFiles">
+ <map key-type="java.lang.String" value-type="java.lang.String">
+ <entry key="../examples/filemgr.properties" value="/etc/filemgr.properties"/>
+ </map>
+ </property>
+ <property name="configFiles">
+ <map key-type="java.lang.String" value-type="java.lang.String">
+ <entry key="../examples/mime-types.xml" value="/etc/mime-types.xml"/>
+ </map>
+ </property>
+ </bean>
+
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/etc/log4j.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/etc/log4j.xml b/config/src/main/resources/etc/log4j.xml
new file mode 100644
index 0000000..8f49451
--- /dev/null
+++ b/config/src/main/resources/etc/log4j.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%-5p %c{1} - %m%n"/>
+ </layout>
+ </appender>
+
+ <root>
+ <priority value="INFO"/>
+ <appender-ref ref="console"/>
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/examples/filemgr.properties
----------------------------------------------------------------------
diff --git a/config/src/main/resources/examples/filemgr.properties b/config/src/main/resources/examples/filemgr.properties
new file mode 100644
index 0000000..4d8d957
--- /dev/null
+++ b/config/src/main/resources/examples/filemgr.properties
@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Configuration properties for the File Manager
+
+# repository factory
+filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory
+
+# Lucene catalog factory
+filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory
+
+# data transfer factory
+filemgr.datatransfer.factory=org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory
+
+# validation layer factory
+filemgr.validationLayer.factory=org.apache.oodt.cas.filemgr.validation.XMLValidationLayerFactory
+
+# xml rpc client configuration
+org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes=20
+org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes=60
+#org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries=0
+#org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds=3
+
+# mapped data source catalog configuration
+#org.apache.oodt.cas.filemgr.catalog.mappeddatasource.mapFile=/path/to/ops.catalog.typemap.properties
+
+# lucene catalog configuration
+org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=[OODT_HOME]/data/catalog
+org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20
+org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60
+org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60
+org.apache.oodt.cas.filemgr.catalog.lucene.mergeFactor=20
+
+# XML repository manager configuration
+org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[FILEMGR_HOME]/policy/oodt
+
+# XML validation layer configuration
+org.apache.oodt.cas.filemgr.validation.dirs=file:[FILEMGR_HOME]/policy/oodt
+
+# remote data transfer configuration
+org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024
+
+# location of Mime-Type repository
+org.apache.oodt.cas.filemgr.mime.type.repository=[FILEMGR_HOME]/etc/mime-types.xml
+
+
+############ data source configuration ##################################
+#
+# These 3 blocks of config properties can be used to setup a catalog,
+# repository manager, and validation layer based on a RDBMS such as
+# Oracle, MySQL, PostgreSQL, or any others that support a jdbc connection.
+# Just un-comment the following blocks of properties and configure as
+# needed.
+#
+#########################################################################
+
+# datasource catalog config
+#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url
+#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user
+#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass
+#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name
+#org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false
+#org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20
+#org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5
+
+# data source repository manager configuration
+#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url
+#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user
+#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass
+#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name
+
+# data source validation layer configuration
+#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url
+#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user
+#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass
+#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name
+#org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false
+
+
+# tells the file manager system layer to include product instance metadata
+# NOTE: here are the expected field mappings
+#
+# product.getProductId() -> ProductId
+# product.getProductName() -> ProductName
+# product.getProductStructure() -> ProductStructure
+# product.getTransferStatus() -> ProductTransferStatus
+# product.getRootRef() -> ProductRootReference
+
+# for the references returned by product.getProductReferences() the following
+# metadata fields will be added (order will be maintained, such that data store
+# ref at index 0 will map to orig ref at index 0, etc.)
+#
+# ProductDataStoreReferences (list of all data store references:
+# note already translated into path, not URI)
+# ProductOrigReferences (list of all orig references:
+# note already translated into path, not URI)
+# ProductMimeType (list of all references' mime-types)
+# ProductFileSize (list of all references' file sizes)
+
+org.apache.oodt.cas.filemgr.metadata.expandProduct=false
\ No newline at end of file