You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/09/02 15:36:01 UTC
[41/50] [abbrv] oodt git commit: Improvements to filemgr and
config-publisher bash scripts
Improvements to filemgr and config-publisher bash scripts
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9ee8af48
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9ee8af48
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9ee8af48
Branch: refs/heads/development
Commit: 9ee8af48ba45904a44d4e32b3030e8ad7d0bee21
Parents: 57a9235
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Thu Aug 10 18:32:27 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Aug 10 18:32:27 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/oodt/config/Component.java | 4 +--
.../config/ConfigurationManagerFactory.java | 37 ++++++++++++++++----
.../java/org/apache/oodt/config/Constants.java | 3 ++
config/src/main/resources/log4j.xml | 1 -
core/pom.xml | 11 +++---
filemgr/pom.xml | 6 +++-
filemgr/src/main/bin/filemgr | 6 ++--
.../TestDistributedXmlRpcFileManager.java | 1 +
.../distributed/config/filemgr.properties | 8 ++---
9 files changed, 54 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Component.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Component.java b/config/src/main/java/org/apache/oodt/config/Component.java
index e2ea223..2c7123e 100644
--- a/config/src/main/java/org/apache/oodt/config/Component.java
+++ b/config/src/main/java/org/apache/oodt/config/Component.java
@@ -24,8 +24,8 @@ package org.apache.oodt.config;
* @author Imesha Sudasingha
*/
public enum Component {
- FILE_MANAGER("filemgr", "OODT_FILEMGR_HOME"),
- RESOURCE_MANAGER("resmgr", "OODT_RESMGR_HOME");
+ FILE_MANAGER("filemgr", "FILEMGR_HOME"),
+ RESOURCE_MANAGER("resmgr", "RESMGR_HOME");
/** Shorthand name of the component. Will be used when creating ZNodes in zookeeper */
String name;
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
index 9eecfaa..f3a5dc5 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java
@@ -17,6 +17,7 @@
package org.apache.oodt.config;
+import org.apache.oodt.config.Constants.Env;
import org.apache.oodt.config.distributed.DistributedConfigurationManager;
import org.apache.oodt.config.standalone.StandaloneConfigurationManager;
import org.slf4j.Logger;
@@ -24,7 +25,9 @@ import org.slf4j.LoggerFactory;
import java.util.List;
+import static org.apache.oodt.config.Constants.Env.CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ENABLE_DISTRIBUTED_CONFIGURATION;
+import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
/**
* Factory class to be used to get the {@link ConfigurationManager} instances accordingly.
@@ -40,18 +43,38 @@ public class ConfigurationManagerFactory {
}
/**
- * Returns the {@link ConfigurationManager} to be used by the calling class. Whether to use the standalone version or
- * the distributed version of the configuration manager will be determined by the value of the property
+ * Returns the {@link ConfigurationManager} to be used by the calling class. Whether to use the standalone version
+ * or the distributed version of the configuration manager will be determined by the value of the property
* <pre>org.apache.oodt.config.zookeeper == true</pre>
*
- * @param component Name of the OODT component, to which the created configuration manager instance will be providing
- * configuration support
- * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone configuration
- * management
+ * @param component Name of the OODT component, to which the created configuration manager instance will be
+ * providing configuration support
+ * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone
+ * configuration management
* @return ConfigurationManager instance to used by the corresponding component.
*/
public static ConfigurationManager getConfigurationManager(Component component, List<String> propertiesFiles) {
- if (Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION)) {
+ boolean isDistributed = Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION);
+ if (!isDistributed) {
+ String env = System.getenv(Env.ENABLE_DISTRIBUTED_CONFIGURATION);
+ if (env != null) {
+ isDistributed = Boolean.parseBoolean(env);
+ }
+ }
+
+ if (isDistributed) {
+ String connectString = System.getProperty(ZK_CONNECT_STRING);
+ if (connectString == null) {
+ connectString = System.getenv(CONNECT_STRING);
+ }
+
+ if (connectString == null) {
+ throw new IllegalArgumentException(
+ String.format("%s environment variable need to be set for distributed configuration management", CONNECT_STRING));
+ }
+
+ System.setProperty(ZK_CONNECT_STRING, connectString);
+
logger.info("Using distributed configuration management for {}", component);
return new DistributedConfigurationManager(component);
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 611f81e..9cd217f 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -43,6 +43,9 @@ public class Constants {
public static class Env {
/** Environment variable name to specify OODT project name */
public static final String OODT_PROJECT = "OODT_PROJECT";
+ /** The environment variable to be set, if to enable distributed configuration management */
+ public static final String ENABLE_DISTRIBUTED_CONFIGURATION = "OODT_DISTRIBUTED_CONF";
+ public static final String CONNECT_STRING = "ZK_CONNECT_STRING";
}
public static class Properties {
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/log4j.xml b/config/src/main/resources/log4j.xml
index 8cc1919..0cd8348 100644
--- a/config/src/main/resources/log4j.xml
+++ b/config/src/main/resources/log4j.xml
@@ -39,7 +39,6 @@
<root>
<priority value="INFO"/>
<appender-ref ref="console"/>
- <appender-ref ref="file"/>
</root>
<logger name="org.apache.zookeeper">
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index b9f07e1..5e99f9e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -253,6 +253,11 @@ the License.
<version>3.3.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.5.1-alpha</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxrs</artifactId>
<version>2.6.0</version>
@@ -412,12 +417,6 @@ the License.
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>6.2.1</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 4fadcb8..2086dc1 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -163,6 +163,10 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-lucene-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -236,7 +240,7 @@
</property>
</systemProperties>
<environmentVariables>
- <OODT_FILEMGR_HOME>${project.basedir}</OODT_FILEMGR_HOME>
+ <FILEMGR_HOME>${project.basedir}</FILEMGR_HOME>
<OODT_PROJECT>primary</OODT_PROJECT>
</environmentVariables>
<forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds>
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/main/bin/filemgr
----------------------------------------------------------------------
diff --git a/filemgr/src/main/bin/filemgr b/filemgr/src/main/bin/filemgr
index 1a4af47..02f5b90 100644
--- a/filemgr/src/main/bin/filemgr
+++ b/filemgr/src/main/bin/filemgr
@@ -34,7 +34,7 @@ else
fi
export JAVA_HOME
-FILEMGR_HOME=..
+FILEMGR_HOME=`cd ..; pwd`
export FILEMGR_HOME
RUN_HOME=${FILEMGR_HOME}/../run
export RUN_HOME
@@ -52,6 +52,8 @@ for file in `find ../lib/*.jar`; do
LIB_DEPS="${file}:${LIB_DEPS}"
done
+LIB_DEPS="${FILEMGR_HOME}/etc/log4j.xml:${LIB_DEPS}"
+
if [ ! -z $OODT_DISTRIBUTED_CONF ]
then
if [ ! -z $ZK_CONNECT_STRING ]
@@ -73,7 +75,7 @@ case "$1" in
$JAVA_HOME/bin/java \
-cp ${LIB_DEPS} \
${DISTRIBUTED_CONF_PROPERTIES} \
- -Dlog4j.configuration=./${FILEMGR_HOME}/etc/log4j.xml \
+ -Dlog4j.configuration=log4j.xml \
-Djava.util.logging.config.file=${FILEMGR_HOME}/etc/logging.properties \
-Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \
org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
index afbac03..3dd158b 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java
@@ -44,6 +44,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+// TODO: 8/10/17 Cleanup Dirs once done
public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigurationTest {
private static final int FM_PORT = 9001;
http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/resources/distributed/config/filemgr.properties
----------------------------------------------------------------------
diff --git a/filemgr/src/test/resources/distributed/config/filemgr.properties b/filemgr/src/test/resources/distributed/config/filemgr.properties
index 50d09a7..2ca5096 100644
--- a/filemgr/src/test/resources/distributed/config/filemgr.properties
+++ b/filemgr/src/test/resources/distributed/config/filemgr.properties
@@ -63,7 +63,7 @@ org.apache.oodt.cas.filemgr.catalog.science.jdbc.pass=
org.apache.oodt.cas.filemgr.catalog.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver
# lucene catalog configuration
-org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=file:target/tmp
+org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=target/tmp
org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20
org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60
org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60
@@ -90,10 +90,10 @@ org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.pass=
org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver
# XML repository manager configuration
-org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt
+org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt
# XML validation layer configuration
-org.apache.oodt.cas.filemgr.validation.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt
+org.apache.oodt.cas.filemgr.validation.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt
# set the following property to 'true' to allow dynamic metadata fields,
# effectively bypassing the validation layer.
@@ -125,7 +125,7 @@ org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
# location of Mime-Type repository
-org.apache.oodt.cas.filemgr.mime.type.repository=[OODT_FILEMGR_HOME]/target/filemgr/policy/mime-types.xml
+org.apache.oodt.cas.filemgr.mime.type.repository=[FILEMGR_HOME]/target/filemgr/policy/mime-types.xml
# tells the file manager system layer to include product instance metadata
# NOTE: here are the expected field mappings