You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2014/07/24 08:07:27 UTC
git commit: OOZIE-1915 Move system properties to conf properties
(puru via rohini)
Repository: oozie
Updated Branches:
refs/heads/master 805c5978e -> 0695c2033
OOZIE-1915 Move system properties to conf properties (puru via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0695c203
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0695c203
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0695c203
Branch: refs/heads/master
Commit: 0695c20337283327fe700b136d27b7a2bc0e6846
Parents: 805c597
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Wed Jul 23 23:07:07 2014 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Wed Jul 23 23:07:07 2014 -0700
----------------------------------------------------------------------
.../oozie/service/ConfigurationService.java | 22 ++++++++++++++------
.../oozie/service/JobsConcurrencyService.java | 8 +++----
.../apache/oozie/servlet/BaseAdminServlet.java | 5 ++++-
.../java/org/apache/oozie/util/ConfigUtils.java | 5 +++--
.../java/org/apache/oozie/util/ZKUtils.java | 2 +-
core/src/main/resources/oozie-default.xml | 11 +++++++++-
.../service/TestJobsConcurrencyService.java | 17 ++++++++-------
.../java/org/apache/oozie/test/ZKXTestCase.java | 2 +-
docs/src/site/twiki/AG_Install.twiki | 16 +++++++++-----
release-log.txt | 1 +
10 files changed, 59 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/service/ConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java
index 798782e..8da50c3 100644
--- a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java
+++ b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java
@@ -81,26 +81,27 @@ public class ConfigurationService implements Service, Instrumentable {
public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
+ private static final Set<String> CONF_SYS_PROPS = new HashSet<String>();
+
private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
private static final Set<String> MASK_PROPS = new HashSet<String>();
static {
- IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
//all this properties are seeded as system properties, no need to log changes
- IGNORE_SYS_PROPS.add("oozie.http.hostname");
- IGNORE_SYS_PROPS.add("oozie.http.port");
- IGNORE_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
-
+ IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
-
IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
+ CONF_SYS_PROPS.add("oozie.http.hostname");
+ CONF_SYS_PROPS.add("oozie.http.port");
+ CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
+
// These properties should be masked when displayed because they contain sensitive info (e.g. password)
MASK_PROPS.add(JPAService.CONF_PASSWORD);
MASK_PROPS.add("oozie.authentication.signature.secret");
@@ -255,6 +256,15 @@ public class ConfigurationService implements Service, Instrumentable {
}
}
+ //Backward compatible, we should still support -Dparam.
+ for (String key : CONF_SYS_PROPS) {
+ String sysValue = System.getProperty(key);
+ if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
+ log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue);
+ configuration.set(key, sysValue);
+ }
+ }
+
return new LogChangesConfiguration(configuration);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java b/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java
index b44d9d7..7508b21 100644
--- a/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java
+++ b/core/src/main/java/org/apache/oozie/service/JobsConcurrencyService.java
@@ -33,11 +33,7 @@ import org.apache.oozie.util.ZKUtils;
*/
public class JobsConcurrencyService implements Service, Instrumentable {
- private static final Map<String, String> urls;
- static {
- urls = new HashMap<String, String>();
- urls.put(System.getProperty(ZKUtils.OOZIE_INSTANCE_ID), ConfigUtils.getOozieEffectiveUrl());
- }
+ private static Map<String, String> urls;
/**
* Initialize the jobs concurrency service
@@ -46,6 +42,8 @@ public class JobsConcurrencyService implements Service, Instrumentable {
*/
@Override
public void init(Services services) throws ServiceException {
+ urls = new HashMap<String, String>();
+ urls.put(services.getConf().get(ZKUtils.OOZIE_INSTANCE_ID), ConfigUtils.getOozieEffectiveUrl());
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
index 128347e..0e44bdf 100644
--- a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
@@ -23,9 +23,11 @@ import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Pattern;
+
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import org.apache.hadoop.fs.Path;
import org.apache.oozie.BuildInfo;
import org.apache.oozie.client.rest.JsonBean;
@@ -38,6 +40,7 @@ import org.apache.oozie.service.JobsConcurrencyService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.util.AuthUrlClient;
+import org.apache.oozie.util.ConfigUtils;
import org.apache.oozie.util.Instrumentation;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -261,7 +264,7 @@ public abstract class BaseAdminServlet extends JsonRestServlet {
private JSONObject updateLocalShareLib(HttpServletRequest request) {
ShareLibService shareLibService = Services.get().get(ShareLibService.class);
JSONObject json = new JSONObject();
- json.put(JsonTags.SHARELIB_UPDATE_HOST, Services.get().getConf().get("oozie.base.url"));
+ json.put(JsonTags.SHARELIB_UPDATE_HOST, ConfigUtils.getOozieEffectiveUrl());
try {
json.putAll(shareLibService.updateShareLib());
json.put(JsonTags.SHARELIB_UPDATE_STATUS, "Successful");
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/util/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ConfigUtils.java b/core/src/main/java/org/apache/oozie/util/ConfigUtils.java
index cdf6b48..c3ac01a 100644
--- a/core/src/main/java/org/apache/oozie/util/ConfigUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ConfigUtils.java
@@ -18,6 +18,7 @@
package org.apache.oozie.util;
import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.service.Services;
import org.apache.oozie.servlet.ServicesLoader;
/**
@@ -83,9 +84,9 @@ public class ConfigUtils {
else {
sb.append("http://");
}
- sb.append(System.getProperty("oozie.http.hostname"));
+ sb.append(Services.get().getConf().get("oozie.http.hostname"));
sb.append(":");
- sb.append(System.getProperty("oozie.http.port"));
+ sb.append(Services.get().getConf().get("oozie.http.port"));
sb.append("/oozie");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/java/org/apache/oozie/util/ZKUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ZKUtils.java b/core/src/main/java/org/apache/oozie/util/ZKUtils.java
index 74f6266..25ed2ee 100644
--- a/core/src/main/java/org/apache/oozie/util/ZKUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ZKUtils.java
@@ -118,7 +118,7 @@ public class ZKUtils {
*/
private ZKUtils() throws Exception {
log = XLog.getLog(getClass());
- zkId = System.getProperty(OOZIE_INSTANCE_ID);
+ zkId = Services.get().getConf().get(OOZIE_INSTANCE_ID, Services.get().getConf().get("oozie.http.hostname"));
createClient();
advertiseService();
checkAndSetACLs();
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 3cb66e7..029ad1e 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -2065,7 +2065,16 @@
</description>
</property>
-
+<!--
+ <property>
+ <name>oozie.instance.id</name>
+ <value>${OOZIE_HTTP_HOSTNAME}</value>
+ <description>
+ Each Oozie server should have its own unique instance id. The default is system property
+ =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname).
+ </description>
+ </property>
+-->
<!-- Sharelib Configuration -->
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java b/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java
index 53df650..473e120 100644
--- a/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestJobsConcurrencyService.java
@@ -31,6 +31,7 @@ public class TestJobsConcurrencyService extends XTestCase {
@Override
protected void setUp() throws Exception {
super.setUp();
+ new Services().init();
}
@Override
@@ -41,7 +42,7 @@ public class TestJobsConcurrencyService extends XTestCase {
public void testIsLeader() throws Exception {
JobsConcurrencyService jcs = new JobsConcurrencyService();
try {
- jcs.init(null);
+ jcs.init(Services.get());
assertTrue(jcs.isLeader());
}
finally {
@@ -52,7 +53,7 @@ public class TestJobsConcurrencyService extends XTestCase {
public void testIsJobIdForThisServer() throws Exception {
JobsConcurrencyService jcs = new JobsConcurrencyService();
try {
- jcs.init(null);
+ jcs.init(Services.get());
assertTrue(jcs.isJobIdForThisServer("blah"));
}
finally {
@@ -63,7 +64,7 @@ public class TestJobsConcurrencyService extends XTestCase {
public void testGetJobIdsForThisServer() throws Exception {
JobsConcurrencyService jcs = new JobsConcurrencyService();
try {
- jcs.init(null);
+ jcs.init(Services.get());
List<String> ids = new ArrayList<String>();
ids.add("blah");
ids.add("0000002-130521183438837-oozie-rkan-W");
@@ -80,11 +81,11 @@ public class TestJobsConcurrencyService extends XTestCase {
public void testGetServerUrls() throws Exception {
JobsConcurrencyService jcs = new JobsConcurrencyService();
try {
- jcs.init(null);
+ jcs.init(Services.get());
Map<String, String> map = jcs.getServerUrls();
assertEquals(1, map.size());
- assertEquals(System.getProperty("oozie.instance.id"), map.keySet().iterator().next());
- assertEquals(ConfigUtils.getOozieURL(false), map.get(System.getProperty("oozie.instance.id")));
+ assertEquals(Services.get().getConf().get("oozie.instance.id"), map.keySet().iterator().next());
+ assertEquals(ConfigUtils.getOozieURL(false), map.get(Services.get().getConf().get("oozie.instance.id")));
}
finally {
jcs.destroy();
@@ -94,7 +95,7 @@ public class TestJobsConcurrencyService extends XTestCase {
public void testsAllServerRequest() throws Exception {
JobsConcurrencyService jcs = new JobsConcurrencyService();
try {
- jcs.init(null);
+ jcs.init(Services.get());
assertFalse(jcs.isAllServerRequest(null));
Map<String, String[]> param = new HashMap<String, String[]>();
assertFalse(jcs.isAllServerRequest(param));
@@ -110,4 +111,4 @@ public class TestJobsConcurrencyService extends XTestCase {
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java b/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java
index 6a2dd32..4c7d0d7 100644
--- a/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/ZKXTestCase.java
@@ -87,7 +87,7 @@ public abstract class ZKXTestCase extends XDataTestCase {
private void setUpZK() throws Exception {
zkServer = setupZKServer();
Services.get().getConf().set("oozie.zookeeper.connection.string", zkServer.getConnectString());
- setSystemProperty("oozie.instance.id", ZK_ID);
+ Services.get().getConf().set("oozie.instance.id", ZK_ID);
createClient();
createServiceDiscovery();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
index 8b0d110..06659f8 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -798,17 +798,23 @@ their own HA, they should have their own namespace. The default value is shown
</property>
</verbatim>
-5. Change the value of =OOZIE_BASE_URL= in oozie-env.sh to point to the loadbalancer or virtual IP, for example:
+5. Change the value of =OOZIE_BASE_URL= in oozie-site.xml to point to the loadbalancer or virtual IP, for example:
<verbatim>
-export OOZIE_BASE_URL="http://my.loadbalancer.hostname:11000/oozie"
+<property>
+ <name>oozie.base.url</name>
+ <value>http://my.loadbalancer.hostname:11000/oozie</value>
+</property>
</verbatim>
-6. (Optional) Change the value of OOZIE_INSTANCE_ID in oozie-env.sh. Each Oozie server should have its own unique instance id. The
-default is =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname).
+6. (Optional) Add the following property to oozie-site.xml in all Oozie servers to specify the each host instance id.
+Each Oozie server in HA should have its own unique instance id. The default is =${OOZIE_HTTP_HOSTNAME}= (i.e. the hostname).
<verbatim>
-export OOZIE_INSTANCE_ID="${OOZIE_HTTP_HOSTNAME}"
+<property>
+ <name>oozie.instance.id</name>
+ <value>hostname</value>
+</property>
</verbatim>
7. (Optional) If using a secure cluster, see [[AG_Install#Security][Security]] below on configuring Kerberos with Oozie HA.
http://git-wip-us.apache.org/repos/asf/oozie/blob/0695c203/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d048bca..70a7dfa 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -4,6 +4,7 @@ OOZIE-1943 Bump up trunk to 4.2.0-SNAPSHOT (bzhang)
-- Oozie 4.1.0 release (4.1 - unreleased)
+OOZIE-1915 Move system properties to conf properties (puru via rohini)
OOZIE-1934 coordinator action repeatedly picked up by cachePurgeWorker of PartitionDependencyManagerService (ryota)
OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona)
OOZIE-1935 Log level (ActionStartXCommand) and Doc fix (CoordinatorFunctionalSpec) (mona)