You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2017/08/21 20:03:08 UTC
metron git commit: METRON-1047 REST should use core-site.xml for
Hadoop configuration (merrimanr) closes apache/metron#672
Repository: metron
Updated Branches:
refs/heads/master ae1dc3a4a -> 222685760
METRON-1047 REST should use core-site.xml for Hadoop configuration (merrimanr) closes apache/metron#672
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/22268576
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/22268576
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/22268576
Branch: refs/heads/master
Commit: 22268576039babc1dd9ee65a5a6a11ade49a7258
Parents: ae1dc3a
Author: merrimanr <me...@gmail.com>
Authored: Mon Aug 21 15:02:57 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Mon Aug 21 15:02:57 2017 -0500
----------------------------------------------------------------------
.../common-services/METRON/CURRENT/package/templates/metron.j2 | 2 +-
.../metron-rest/src/main/config/rest_application.yml | 4 ----
.../main/java/org/apache/metron/rest/MetronRestConstants.java | 5 -----
.../main/java/org/apache/metron/rest/config/HadoopConfig.java | 2 --
metron-interface/metron-rest/src/main/scripts/metron-rest | 3 ++-
.../java/org/apache/metron/rest/config/HadoopConfigTest.java | 6 ------
6 files changed, 3 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index 06dddd3..7233b54 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -31,7 +31,7 @@ METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}"
METRON_SPRING_OPTIONS="{{metron_spring_options}}"
ZOOKEEPER="{{zookeeper_quorum}}"
BROKERLIST="{{kafka_brokers}}"
-HDFS_URL="{{default_fs}}"
+HADOOP_CONF_DIR="/etc/hadoop/conf/"
STORM_REST_URL="{{storm_rest_addr}}"
SECURITY_ENABLED={{security_enabled|lower}}
{% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}"
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 2afbb8a..6d12e95 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -31,10 +31,6 @@ kafka:
security:
protocol: ${KAFKA_SECURITY_PROTOCOL}
-hdfs:
- namenode:
- url: ${HDFS_URL}
-
grok:
path:
temp: ${METRON_TEMP_GROK_PATH}
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index a080f77..11310d4 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -30,8 +30,6 @@ public class MetronRestConstants {
public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp";
public static final String GROK_CLASS_NAME = GrokParser.class.getName();
public static final String GROK_PATH_KEY = "grokPath";
- public static final String GROK_STATEMENT_KEY = "grokStatement";
- public static final String GROK_PATTERN_LABEL_KEY = "patternLabel";
public static final String STORM_UI_SPRING_PROPERTY = "storm.ui.url";
public static final String SUPERVISOR_SUMMARY_URL = "/api/v1/supervisor/summary";
@@ -53,9 +51,6 @@ public class MetronRestConstants {
public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url";
- public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url";
- public static final String DEFAULT_HDFS_URL = "file:///";
-
public static final String KERBEROS_ENABLED_SPRING_PROPERTY = "kerberos.enabled";
public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY = "kerberos.principal";
public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab";
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
index 7b1bf2f..98c8108 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
@@ -39,9 +39,7 @@ public class HadoopConfig {
@Bean
public org.apache.hadoop.conf.Configuration configuration() throws IOException {
org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
- configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL));
if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) {
- configuration.set("hadoop.security.authentication", "KERBEROS");
UserGroupInformation.setConfiguration(configuration);
String keyTabLocation = environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY);
String userPrincipal = environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY);
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/scripts/metron-rest
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest
index 08cdfd4..0d99617 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest
@@ -46,8 +46,9 @@ fi
HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
PIDFILE="$METRON_PID_DIR/$NAME.pid"
+LOCKFILE=/var/lock/subsys/$NAME
-METRON_REST_CLASSPATH="${HBASE_HOME}/conf:$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar"
+METRON_REST_CLASSPATH="$HADOOP_CONF_DIR:${HBASE_HOME}/conf:$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar"
# the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml
if [[ !($METRON_SPRING_PROFILES_ACTIVE == *"vagrant"*) ]]; then
http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java
index c262783..7504e3e 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java
@@ -56,7 +56,6 @@ public class HadoopConfigTest {
@Test
public void configurationShouldReturnProperKerberosConfiguration() throws IOException {
- when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem");
when(environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY)).thenReturn("metron keytabLocation");
when(environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY)).thenReturn("metron principal");
@@ -67,14 +66,10 @@ public class HadoopConfigTest {
verifyStatic();
UserGroupInformation.setConfiguration(any(Configuration.class));
UserGroupInformation.loginUserFromKeytab("metron keytabLocation", "metron principal");
-
- assertEquals("default filesystem", configuration.get("fs.defaultFS"));
- assertEquals("KERBEROS", configuration.get("hadoop.security.authentication"));
}
@Test
public void configurationShouldReturnProperConfiguration() throws IOException {
- when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem");
when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(false);
Configuration configuration = hadoopConfig.configuration();
@@ -83,7 +78,6 @@ public class HadoopConfigTest {
UserGroupInformation.setConfiguration(any(Configuration.class));
UserGroupInformation.loginUserFromKeytab(anyString(), anyString());
- assertEquals("default filesystem", configuration.get("fs.defaultFS"));
assertEquals("simple", configuration.get("hadoop.security.authentication"));
}
}