You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/09/02 15:35:46 UTC
[26/50] [abbrv] oodt git commit: Added missing conf-publisher binary
and minor improvements
Added missing conf-publisher binary and minor improvements
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/39b6c888
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/39b6c888
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/39b6c888
Branch: refs/heads/development
Commit: 39b6c888512b74feaa9d5494c7bd7268f4b42f6e
Parents: 8b02792
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Jul 26 21:51:41 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Fri Jul 28 21:20:43 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 36 +-
config/src/main/assembly/assembly.xml | 4 +-
.../DistributedConfigurationManager.java | 1 -
.../DistributedConfigurationPublisher.java | 257 ++++++++++++++
.../config/distributed/cli/ConfigPublisher.java | 105 ++++++
.../cli/DistributedConfigurationPublisher.java | 331 -------------------
.../config/distributed/utils/FilePathUtils.java | 6 +-
.../src/main/resources/etc/config-publisher.xml | 16 +-
config/src/main/resources/etc/log4j.xml | 5 +-
config/src/main/scripts/conf-publisher | 36 ++
.../DistributedConfigurationManagerTest.java | 25 +-
.../DistributedConfigurationPublisherTest.java | 6 +-
.../src/test/resources/etc/config-publisher.xml | 4 +-
13 files changed, 454 insertions(+), 378 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/pom.xml
----------------------------------------------------------------------
diff --git a/config/pom.xml b/config/pom.xml
index 741caac..6719c48 100644
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>oodt-core</artifactId>
<groupId>org.apache.oodt</groupId>
- <version>1.1-SNAPSHOT</version>
+ <version>1.2-SNAPSHOT</version>
<relativePath>../core/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -17,29 +17,25 @@
<dependencies>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
-
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-beans</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context</artifactId>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
</dependency>
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
</dependency>
<dependency>
- <groupId>args4j</groupId>
- <artifactId>args4j</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -50,12 +46,16 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/config/src/main/assembly/assembly.xml b/config/src/main/assembly/assembly.xml
index 2743cab..bb6f31e 100644
--- a/config/src/main/assembly/assembly.xml
+++ b/config/src/main/assembly/assembly.xml
@@ -33,8 +33,8 @@
</includes>
</fileSet>
<fileSet>
- <directory>${basedir}/src/main/bin</directory>
- <outputDirectory>bin</outputDirectory>
+ <directory>${basedir}/src/main/scripts</directory>
+ <outputDirectory>.</outputDirectory>
<includes/>
<fileMode>755</fileMode>
</fileSet>
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index da96913..122a78e 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -37,7 +37,6 @@ import java.util.concurrent.TimeUnit;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
-import static org.apache.oodt.config.Constants.SEPARATOR;
/**
* Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
new file mode 100644
index 0000000..81fe2af
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config.distributed;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.oodt.config.Component;
+import org.apache.oodt.config.Constants;
+import org.apache.oodt.config.distributed.utils.CuratorUtils;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
+import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
+
+/**
+ * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per
+ * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that.
+ *
+ * @author Imesha Sudasingha
+ */
+public class DistributedConfigurationPublisher {
+
+ private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class);
+
+ private Map<String, String> propertiesFiles;
+ private Map<String, String> configFiles;
+ private String connectString;
+ private CuratorFramework client;
+ private ZNodePaths zNodePaths;
+ private Component component;
+
+ public DistributedConfigurationPublisher(Component component) {
+ this.component = component;
+ this.zNodePaths = new ZNodePaths(this.component.getName());
+
+ if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
+ }
+
+ if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
+ try {
+ CuratorUtils.loadZookeeperProperties();
+ } catch (IOException e) {
+ logger.error("Error occurred when loading properties from properties file");
+ }
+ }
+
+ if (System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
+ }
+
+ connectString = System.getProperty(ZK_CONNECT_STRING);
+ logger.info("Using zookeeper connect string : {}", connectString);
+
+ startZookeeper();
+ }
+
+ /**
+ * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of {@link
+ * Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
+ */
+ private void startZookeeper() {
+ client = CuratorUtils.newCuratorFrameworkClient(connectString, logger);
+
+ client.start();
+ logger.info("Curator framework start operation invoked");
+
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
+ try {
+ logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
+ client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ex) {
+ logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString);
+ }
+
+ if (!client.getZookeeperClient().isConnected()) {
+ throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString);
+ }
+
+ logger.info("CuratorFramework client started successfully");
+ }
+
+ public void destroy() {
+ logger.debug("Destroying configuration publisher");
+ try {
+ client.close();
+ } catch (Exception e) {
+ logger.error("Error occurred when trying to close Curator client : {}", e);
+ }
+
+ logger.info("Configuration publisher destroyed");
+ }
+
+ /**
+ * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being
+ * published, no further publishing attempts will be carried on. Error will be reported to user.
+ *
+ * @throws Exception Zookeeper errors
+ */
+ public void publishConfiguration() throws Exception {
+ logger.debug("Publishing properties files : {}", propertiesFiles);
+ publishConfiguration(propertiesFiles, true);
+ logger.info("Properties files published successfully");
+
+ logger.debug("Publishing config files : {}", configFiles);
+ publishConfiguration(configFiles, false);
+ logger.info("Config files published successfully");
+ }
+
+ /**
+ * Verified whether the actual content of the local files specified to be published are 100% similar to the ones
+ * that has been published and stored in zookeeper at the moment.
+ *
+ * @return true | if content are up to date and similar
+ */
+ public boolean verifyPublishedConfiguration() {
+ try {
+ return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false);
+ } catch (Exception e) {
+ logger.error("Error occurred when checking published config", e);
+ return false;
+ }
+ }
+
+ /**
+ * Removes all the nodes from zookeeper where the configuration corresponding to component {@link #component} is
+ * stored
+ *
+ * @throws Exception zookeeper errors
+ */
+ public void clearConfiguration() throws Exception {
+ logger.debug("Clearing configuration from zookeeper");
+ CuratorUtils.deleteChildNodes(client, zNodePaths.getPropertiesZNodePath());
+ CuratorUtils.deleteChildNodes(client, zNodePaths.getConfigurationZNodePath());
+ logger.info("Configuration cleared!");
+ }
+
+ private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
+ for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
+ String filePath = entry.getKey();
+ String relativeZNodePath = entry.getValue();
+ logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath);
+
+ String content = getFileContent(filePath);
+
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ if (client.checkExists().forPath(zNodePath) != null) {
+ byte[] bytes = client.getData().forPath(zNodePath);
+ String existingData = new String(bytes);
+ if (existingData.equals(content)) {
+ logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath);
+ } else {
+ Stat stat = client.setData().forPath(zNodePath, content.getBytes());
+ if (stat != null) {
+ logger.info("Published configuration file {} to {}", filePath, relativeZNodePath);
+ } else {
+ logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath);
+ }
+ }
+ } else {
+ /*
+ * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted
+ * when no child node is present under them.
+ */
+ client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes());
+ logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath);
+ }
+ }
+ }
+
+ private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
+ boolean noError = true;
+ for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
+ String filePath = entry.getKey();
+ String relativeZNodePath = entry.getValue();
+ logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath);
+
+ String originalContent = getFileContent(filePath);
+
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ if (client.checkExists().forPath(zNodePath) == null) {
+ logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath);
+ noError = false;
+ continue;
+ }
+
+ String publishedContent = new String(client.getData().forPath(zNodePath));
+ if (!publishedContent.equals(originalContent)) {
+ logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath);
+ noError = false;
+ continue;
+ }
+
+ logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath);
+ }
+
+ return noError;
+ }
+
+ private String getFileContent(String file) {
+ String content;
+ try {
+ content = FileUtils.readFileToString(new File(file));
+ } catch (IOException e) {
+ logger.error("Unable to read file : {}", file, e);
+ throw new IllegalArgumentException("Unable to read content of the file : " + file);
+ }
+
+ return content;
+ }
+
+ public Map<String, String> getPropertiesFiles() {
+ return propertiesFiles;
+ }
+
+ public void setPropertiesFiles(Map<String, String> propertiesFiles) {
+ this.propertiesFiles = propertiesFiles;
+ }
+
+ public Map<String, String> getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(Map<String, String> configFiles) {
+ this.configFiles = configFiles;
+ }
+
+ public ZNodePaths getZNodePaths() {
+ return zNodePaths;
+ }
+
+ public Component getComponent() {
+ return component;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
new file mode 100644
index 0000000..7a78a8c
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config.distributed.cli;
+
+import org.apache.oodt.config.Constants;
+import org.apache.oodt.config.distributed.DistributedConfigurationPublisher;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.Map;
+
+import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
+
+public class ConfigPublisher {
+
+ private static final Logger logger = LoggerFactory.getLogger(ConfigPublisher.class);
+
+ public static void main(String[] args) throws Exception {
+ CmdLineOptions cmdLineOptions = new CmdLineOptions();
+ CmdLineParser parser = new CmdLineParser(cmdLineOptions);
+
+ try {
+ parser.parseArgument(args);
+ } catch (CmdLineException e) {
+ System.err.println("There's an error in your command");
+ parser.printUsage(System.err);
+ return;
+ }
+
+ if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ System.err.println("Zookeeper connect string is not found");
+ parser.printUsage(System.err);
+ return;
+ } else {
+ System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString());
+ }
+
+ System.out.println("Starting configuration publishing");
+
+ try {
+ ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML);
+ Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
+
+ for (Object bean : distributedConfigurationPublisher.values()) {
+ DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+ System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponent()));
+
+ if (cmdLineOptions.isPublish()) {
+ System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent()));
+ publisher.publishConfiguration();
+ System.out.println(String.format("Published configuration for : %s", publisher.getComponent()));
+ System.out.println();
+ }
+
+ if (cmdLineOptions.isVerify()) {
+ System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent()));
+ if (publisher.verifyPublishedConfiguration()) {
+ System.out.println("OK... Configuration verified");
+ System.out.println(String.format("Verified configuration for : %s", publisher.getComponent()));
+ } else {
+ System.err.println("ERROR... Published configuration doesn't match the local files. Please check above logs");
+ }
+ System.out.println();
+ }
+
+ if (cmdLineOptions.isClear()) {
+ System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent()));
+ publisher.clearConfiguration();
+ System.out.println(String.format("Cleared configuration for : %s", publisher.getComponent()));
+ System.out.println();
+ }
+
+ publisher.destroy();
+ }
+ } catch (BeansException e) {
+ logger.error("Error occurred when obtaining configuration publisher beans", e);
+ throw e;
+ } catch (Exception e) {
+ logger.error("Error occurred when publishing configuration to zookeeper", e);
+ throw e;
+ }
+
+ logger.info("Exiting CLI ...");
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
deleted file mode 100644
index 25a45ba..0000000
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.config.distributed.cli;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.oodt.config.Component;
-import org.apache.oodt.config.Constants;
-import org.apache.oodt.config.distributed.ZNodePaths;
-import org.apache.oodt.config.distributed.utils.CuratorUtils;
-import org.apache.zookeeper.data.Stat;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
-import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
-
-/**
- * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per
- * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that.
- *
- * @author Imesha Sudasingha
- */
-public class DistributedConfigurationPublisher {
-
- private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class);
-
- private Map<String, String> propertiesFiles;
- private Map<String, String> configFiles;
- private String connectString;
- private CuratorFramework client;
- private ZNodePaths zNodePaths;
- private Component component;
-
- public DistributedConfigurationPublisher(Component component) {
- this.component = component;
- this.zNodePaths = new ZNodePaths(this.component.getName());
-
- if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
- }
-
- if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
- try {
- CuratorUtils.loadZookeeperProperties();
- } catch (IOException e) {
- logger.error("Error occurred when loading properties from properties file");
- }
- }
-
- if (System.getProperty(ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
- }
-
- connectString = System.getProperty(ZK_CONNECT_STRING);
- logger.info("Using zookeeper connect string : {}", connectString);
-
- startZookeeper();
- }
-
- /**
- * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of {@link
- * Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble.
- */
- private void startZookeeper() {
- client = CuratorUtils.newCuratorFrameworkClient(connectString, logger);
-
- client.start();
- logger.info("Curator framework start operation invoked");
-
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000"));
- try {
- logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
- client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
- } catch (InterruptedException ex) {
- logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString);
- }
-
- if (!client.getZookeeperClient().isConnected()) {
- throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString);
- }
-
- logger.info("CuratorFramework client started successfully");
- }
-
- public void destroy() {
- logger.debug("Destroying configuration publisher");
- try {
- client.close();
- } catch (Exception e) {
- logger.error("Error occurred when trying to close Curator client : {}", e);
- }
-
- logger.info("Configuration publisher destroyed");
- }
-
- /**
- * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being
- * published, no further publishing attempts will be carried on. Error will be reported to user.
- *
- * @throws Exception Zookeeper errors
- */
- public void publishConfiguration() throws Exception {
- logger.debug("Publishing properties files : {}", propertiesFiles);
- publishConfiguration(propertiesFiles, true);
- logger.info("Properties files published successfully");
-
- logger.debug("Publishing config files : {}", configFiles);
- publishConfiguration(configFiles, false);
- logger.info("Config files published successfully");
- }
-
- /**
- * Verified whether the actual content of the local files specified to be published are 100% similar to the ones
- * that has been published and stored in zookeeper at the moment.
- *
- * @return true | if content are up to date and similar
- */
- public boolean verifyPublishedConfiguration() {
- try {
- return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false);
- } catch (Exception e) {
- logger.error("Error occurred when checking published config", e);
- return false;
- }
- }
-
- /**
- * Removes all the nodes from zookeeper where the configuration corresponding to component {@link #component} is
- * stored
- *
- * @throws Exception zookeeper errors
- */
- public void clearConfiguration() throws Exception {
- logger.debug("Clearing configuration from zookeeper");
- CuratorUtils.deleteChildNodes(client, zNodePaths.getPropertiesZNodePath());
- CuratorUtils.deleteChildNodes(client, zNodePaths.getConfigurationZNodePath());
- logger.info("Configuration cleared!");
- }
-
- private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
- for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
- String filePath = entry.getKey();
- String relativeZNodePath = entry.getValue();
- logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath);
-
- String content = getFileContent(filePath);
-
- String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
- if (client.checkExists().forPath(zNodePath) != null) {
- byte[] bytes = client.getData().forPath(zNodePath);
- String existingData = new String(bytes);
- if (existingData.equals(content)) {
- logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath);
- } else {
- Stat stat = client.setData().forPath(zNodePath, content.getBytes());
- if (stat != null) {
- logger.info("Published configuration file {} to {}", filePath, relativeZNodePath);
- } else {
- logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath);
- }
- }
- } else {
- /*
- * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted
- * when no child node is present under them.
- */
- client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes());
- logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath);
- }
- }
- }
-
- private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
- boolean noError = true;
- for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
- String filePath = entry.getKey();
- String relativeZNodePath = entry.getValue();
- logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath);
-
- String originalContent = getFileContent(filePath);
-
- String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
- if (client.checkExists().forPath(zNodePath) == null) {
- logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath);
- noError = false;
- continue;
- }
-
- String publishedContent = new String(client.getData().forPath(zNodePath));
- if (!publishedContent.equals(originalContent)) {
- logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath);
- noError = false;
- continue;
- }
-
- logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath);
- }
-
- return noError;
- }
-
- private String getFileContent(String file) {
- String content;
- try {
- content = FileUtils.readFileToString(new File(file));
- } catch (IOException e) {
- logger.error("Unable to read file : {}", file, e);
- throw new IllegalArgumentException("Unable to read content of the file : " + file);
- }
-
- return content;
- }
-
- public Map<String, String> getPropertiesFiles() {
- return propertiesFiles;
- }
-
- public void setPropertiesFiles(Map<String, String> propertiesFiles) {
- this.propertiesFiles = propertiesFiles;
- }
-
- public Map<String, String> getConfigFiles() {
- return configFiles;
- }
-
- public void setConfigFiles(Map<String, String> configFiles) {
- this.configFiles = configFiles;
- }
-
- public ZNodePaths getZNodePaths() {
- return zNodePaths;
- }
-
- public static void main(String[] args) throws Exception {
- CmdLineOptions cmdLineOptions = new CmdLineOptions();
- CmdLineParser parser = new CmdLineParser(cmdLineOptions);
-
- try {
- parser.parseArgument(args);
- } catch (CmdLineException e) {
- System.err.println("There's an error in your command");
- parser.printUsage(System.err);
- return;
- }
-
- if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) {
- System.err.println("Zookeeper connect string is not found");
- parser.printUsage(System.err);
- return;
- } else {
- System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString());
- }
-
- System.out.println("Starting configuration publishing");
-
- try {
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML);
- Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
-
- for (Object bean : distributedConfigurationPublisher.values()) {
- DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
- System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponent()));
-
- if (cmdLineOptions.isPublish()) {
- System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent()));
- publisher.publishConfiguration();
- System.out.println(String.format("Published configuration for : %s", publisher.getComponent()));
- System.out.println();
- }
-
- if (cmdLineOptions.isVerify()) {
- System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent()));
- if (publisher.verifyPublishedConfiguration()) {
- System.out.println("OK... Configuration verified");
- System.out.println(String.format("Verified configuration for : %s", publisher.getComponent()));
- } else {
- System.err.println("ERROR... Published configuration doesn't match the local files. Please check above logs");
- }
- System.out.println();
- }
-
- if (cmdLineOptions.isClear()) {
- System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent()));
- publisher.clearConfiguration();
- System.out.println(String.format("Cleared configuration for : %s", publisher.getComponent()));
- System.out.println();
- }
-
- publisher.destroy();
- }
- } catch (BeansException e) {
- logger.error("Error occurred when obtaining configuration publisher beans", e);
- throw e;
- } catch (Exception e) {
- logger.error("Error occurred when publishing configuration to zookeeper", e);
- throw e;
- }
-
- logger.info("Exiting CLI ...");
- }
-
- public Component getComponent() {
- return component;
- }
-}
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java
index 5b242c2..cfb5ff9 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/FilePathUtils.java
@@ -32,7 +32,11 @@ public class FilePathUtils {
}
public static String fixForComponentHome(Component component, String suffixPath) {
- String prefix = System.getenv().get(component.getHome());
+ String prefix = System.getProperty(component.getHome());
+ if (prefix == null) {
+ prefix = System.getenv().get(component.getHome());
+ }
+
StringBuilder path = new StringBuilder();
if (prefix != null && !prefix.trim().isEmpty()) {
prefix = prefix.trim();
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/resources/etc/config-publisher.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/etc/config-publisher.xml b/config/src/main/resources/etc/config-publisher.xml
index 443a7a5..88ba33b 100644
--- a/config/src/main/resources/etc/config-publisher.xml
+++ b/config/src/main/resources/etc/config-publisher.xml
@@ -20,21 +20,21 @@
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
<!-- Configuration publisher for File Manager OODT Component -->
- <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher">
+ <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
<constructor-arg value="FILE_MANAGER"/>
<property name="propertiesFiles">
<map key-type="java.lang.String" value-type="java.lang.String">
- <entry key="../examples/filemgr/filemgr.properties" value="/etc/filemgr.properties"/>
+ <entry key="examples/filemgr/filemgr.properties" value="/etc/filemgr.properties"/>
</map>
</property>
<property name="configFiles">
<map key-type="java.lang.String" value-type="java.lang.String">
- <entry key="../examples/filemgr/mime-types.xml" value="/etc/mime-types.xml"/>
- <entry key="../examples/filemgr/cmd-line-actions.xml" value="/policy/cmd-line-actions.xml"/>
- <entry key="../examples/filemgr/cmd-line-options.xml" value="/policy/cmd-line-options.xml"/>
- <entry key="../examples/filemgr/oodt/elements.xml" value="/policy/oodt/elements.xml"/>
- <entry key="../examples/filemgr/oodt/product-types.xml" value="/policy/oodt/product-types.xml"/>
- <entry key="../examples/filemgr/oodt/product-type-element-map.xml" value="/policy/oodt/product-type-element-map.xml"/>
+ <entry key="examples/filemgr/mime-types.xml" value="/etc/mime-types.xml"/>
+ <entry key="examples/filemgr/cmd-line-actions.xml" value="/policy/cmd-line-actions.xml"/>
+ <entry key="examples/filemgr/cmd-line-options.xml" value="/policy/cmd-line-options.xml"/>
+ <entry key="examples/filemgr/oodt/elements.xml" value="/policy/oodt/elements.xml"/>
+ <entry key="examples/filemgr/oodt/product-types.xml" value="/policy/oodt/product-types.xml"/>
+ <entry key="examples/filemgr/oodt/product-type-element-map.xml" value="/policy/oodt/product-type-element-map.xml"/>
</map>
</property>
</bean>
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/resources/etc/log4j.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/etc/log4j.xml b/config/src/main/resources/etc/log4j.xml
index 17d6e8e..8cc1919 100644
--- a/config/src/main/resources/etc/log4j.xml
+++ b/config/src/main/resources/etc/log4j.xml
@@ -37,7 +37,7 @@
</appender>
<root>
- <priority value="DEBUG"/>
+ <priority value="INFO"/>
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
@@ -45,6 +45,9 @@
<logger name="org.apache.zookeeper">
<level value="ERROR"/>
</logger>
+ <logger name="org.springframework">
+ <level value="ERROR"/>
+ </logger>
<logger name="org.apache.curator">
<level value="ERROR"/>
</logger>
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/main/scripts/conf-publisher
----------------------------------------------------------------------
diff --git a/config/src/main/scripts/conf-publisher b/config/src/main/scripts/conf-publisher
new file mode 100644
index 0000000..618af4b
--- /dev/null
+++ b/config/src/main/scripts/conf-publisher
@@ -0,0 +1,36 @@
+#!/bin/sh
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one or more
+# * contributor license agreements. See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The ASF licenses this file to You under the Apache License, Version 2.0
+# * (the "License"); you may not use this file except in compliance with
+# * the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+[ -f /etc/sysconfig/java ] && . /etc/sysconfig/java
+
+CONF_HOME=.
+
+for file in `find ${CONF_HOME}/lib/*.jar`; do
+ LIB_DEPS="${file}:${LIB_DEPS}"
+done
+
+LIB_DEPS="${CONF_HOME}/etc/log4j.xml:${LIB_DEPS}"
+
+echo -n "Starting Configuration Publisher"
+$JAVA_HOME/bin/java \
+ -cp ${LIB_DEPS} \
+ -Dlog4j.configuration=etc/log4j.xml \
+ org.apache.oodt.config.distributed.cli.ConfigPublisher "$@"
+echo "OK"
+sleep 1
+exit 0
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
index 08ecdc1..9052ac3 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
@@ -19,14 +19,12 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.oodt.config.ConfigurationManager;
-import org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher;
+import org.apache.oodt.config.distributed.cli.ConfigPublisher;
import org.apache.oodt.config.distributed.utils.FilePathUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.fail;
-import static org.junit.Assert.assertNotNull;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
@@ -42,6 +40,7 @@ import java.util.Set;
import static org.apache.oodt.config.Constants.CONFIG_PUBLISHER_XML;
import static org.apache.oodt.config.Constants.SEPARATOR;
+import static org.junit.Assert.fail;
/**
* Testing the {@link DistributedConfigurationManager} whether it is downloading and storing the configuration correctly
@@ -57,7 +56,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
public static void setUp() throws Exception {
AbstractDistributedConfigurationTest.setUp();
- DistributedConfigurationPublisher.main(new String[]{
+ ConfigPublisher.main(new String[]{
"-connectString", zookeeper.getConnectString(),
"-publish"
});
@@ -68,6 +67,8 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
publishers = new ArrayList<>(distributedConfigurationPublishers.values().size());
for (Object bean : distributedConfigurationPublishers.values()) {
DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+
+ System.setProperty(publisher.getComponent().getHome(), ".");
publishers.add(publisher);
}
}
@@ -84,11 +85,10 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
Properties properties = new Properties();
try (InputStream in = new FileInputStream(originalFile)) {
properties.load(in);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
}
- catch (Exception e){
- e.printStackTrace();
- fail(e.getMessage());
- }
for (String key : properties.stringPropertyNames()) {
Assert.assertEquals(properties.getProperty(key), System.getProperty(key));
@@ -98,7 +98,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
fileName = fileName.startsWith(SEPARATOR) ? fileName.substring(SEPARATOR.length()) : fileName;
fileName = FilePathUtils.fixForComponentHome(publisher.getComponent(), fileName);
File downloadedFile = new File(fileName);
- Assert.assertNotNull(downloadedFile);
+ Assert.assertNotNull(downloadedFile);
Assert.assertTrue(downloadedFile.exists());
}
@@ -126,7 +126,10 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
String fileName = entry.getValue();
fileName = fileName.startsWith(SEPARATOR) ? fileName.substring(1) : fileName;
- String prefixPath = System.getenv(publisher.getComponent().getHome());
+ String prefixPath = System.getProperty(publisher.getComponent().getHome());
+ if (prefixPath == null) {
+ prefixPath = System.getenv(publisher.getComponent().getHome());
+ }
String confDir = prefixPath != null && !prefixPath.trim().isEmpty() ?
prefixPath.trim() + SEPARATOR + fileName.split(SEPARATOR)[0] : fileName.split(SEPARATOR)[0];
@@ -135,7 +138,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
}
}
- DistributedConfigurationPublisher.main(new String[]{
+ ConfigPublisher.main(new String[]{
"-connectString", zookeeper.getConnectString(),
"-clear"
});
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
index 4f1552b..2c130db 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java
@@ -18,7 +18,7 @@
package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
-import org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher;
+import org.apache.oodt.config.distributed.cli.ConfigPublisher;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
@@ -41,7 +41,7 @@ public class DistributedConfigurationPublisherTest extends AbstractDistributedCo
@Test
public void publishConfigurationTest() throws Exception {
// Publishing configuration through CLI and verifying whether they were stored correctly
- DistributedConfigurationPublisher.main(new String[]{
+ ConfigPublisher.main(new String[]{
"-connectString", zookeeper.getConnectString(),
"-publish",
"-verify"
@@ -82,7 +82,7 @@ public class DistributedConfigurationPublisherTest extends AbstractDistributedCo
}
// Clearing configuration through CLI and checking whether the configuration has actually been gone
- DistributedConfigurationPublisher.main(new String[]{
+ ConfigPublisher.main(new String[]{
"-connectString", zookeeper.getConnectString(),
"-clear"
});
http://git-wip-us.apache.org/repos/asf/oodt/blob/39b6c888/config/src/test/resources/etc/config-publisher.xml
----------------------------------------------------------------------
diff --git a/config/src/test/resources/etc/config-publisher.xml b/config/src/test/resources/etc/config-publisher.xml
index 89e16f3..ecae033 100644
--- a/config/src/test/resources/etc/config-publisher.xml
+++ b/config/src/test/resources/etc/config-publisher.xml
@@ -19,7 +19,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
- <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher">
+ <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
<constructor-arg value="FILE_MANAGER"/>
<property name="propertiesFiles">
<map key-type="java.lang.String" value-type="java.lang.String">
@@ -38,7 +38,7 @@
</property>
</bean>
- <bean id="resmgr-config-publisher" class="org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher">
+ <bean id="resmgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
<constructor-arg value="RESOURCE_MANAGER"/>
<property name="propertiesFiles">
<map key-type="java.lang.String" value-type="java.lang.String">