You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/09/08 20:25:38 UTC
[4/5] metron git commit: METRON-1150 REST parseMessage endpoint fails
with Unable to load < grok file>
from either classpath or HDFS (merrimanr) closes apache/metron#729
METRON-1150 REST parseMessage endpoint fails with Unable to load <grok file> from either classpath or HDFS (merrimanr) closes apache/metron#729
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/224d3d5e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/224d3d5e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/224d3d5e
Branch: refs/heads/Metron_0.4.1
Commit: 224d3d5e2b8c17ca4414ad7d7846dafee0b06d3b
Parents: f79214b
Author: merrimanr <me...@gmail.com>
Authored: Fri Sep 8 09:43:29 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Fri Sep 8 09:43:29 2017 -0500
----------------------------------------------------------------------
.../CURRENT/configuration/metron-rest-env.xml | 10 +---
.../package/scripts/params/params_linux.py | 6 ++-
.../METRON/CURRENT/package/templates/metron.j2 | 1 -
.../src/main/config/rest_application.yml | 1 -
.../apache/metron/rest/MetronRestConstants.java | 1 -
.../apache/metron/rest/service/GrokService.java | 6 ++-
.../apache/metron/rest/service/HdfsService.java | 2 +
.../rest/service/impl/GrokServiceImpl.java | 37 ++++++-------
.../rest/service/impl/HdfsServiceImpl.java | 9 ++++
.../impl/SensorParserConfigServiceImpl.java | 41 +++++++-------
.../src/main/resources/application-docker.yml | 1 -
.../src/main/resources/application-test.yml | 1 -
.../src/main/resources/application-vagrant.yml | 3 +-
.../src/main/resources/application.yml | 3 +-
...orParserConfigControllerIntegrationTest.java | 6 ---
.../rest/service/impl/GrokServiceImplTest.java | 56 +++++++-------------
.../impl/SensorParserConfigServiceImplTest.java | 20 +++++--
17 files changed, 97 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 9c11123..0549510 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -90,17 +90,11 @@
</property>
<property>
<name>metron_temp_grok_path</name>
- <description>Temporary local file path where grok patterns are written during testing</description>
- <value>./patterns/temp</value>
+ <description>Temporary HDFS file path where grok patterns are written during testing</description>
+ <value>{{metron_apps_hdfs_dir}}/patterns/tmp</value>
<display-name>Metron temp grok path</display-name>
</property>
<property>
- <name>metron_default_grok_path</name>
- <description>Default HDFS directory path used when storing Grok patterns</description>
- <value>/apps/metron/patterns</value>
- <display-name>Metron default grok path</display-name>
- </property>
- <property>
<name>metron_spring_options</name>
<description>Additional Spring options not included in the rest_application.yml file</description>
<display-name>Metron Spring options</display-name>
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 78d253c..a9d00dd 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -55,8 +55,6 @@ metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_
metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password']
metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform']
metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path']
-metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path']
-metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path']
metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options']
metron_escalation_topic = config['configurations']['metron-rest-env']['metron_escalation_topic']
metron_config_path = metron_home + '/config'
@@ -132,6 +130,10 @@ if has_kafka_host:
metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
+# well enough until we find a better way via Ambari
+metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path']))
+
metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
local_grok_patterns_dir = format("{metron_home}/patterns")
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index dd37946..049b6ee 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -27,7 +27,6 @@ METRON_JDBC_USERNAME="{{metron_jdbc_username}}"
METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}"
METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}"
METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}"
-METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}"
METRON_SPRING_OPTIONS="{{metron_spring_options}}"
ZOOKEEPER="{{zookeeper_quorum}}"
BROKERLIST="{{kafka_brokers}}"
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 0c17580..6e4fb66 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -36,7 +36,6 @@ kafka:
grok:
path:
temp: ${METRON_TEMP_GROK_PATH}
- default: ${METRON_DEFAULT_GROK_PATH}
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 7c9cdac..7f8110c 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -26,7 +26,6 @@ public class MetronRestConstants {
public static final String DOCKER_PROFILE = "docker";
public static final String CSRF_ENABLE_PROFILE = "csrf-enable";
- public static final String GROK_DEFAULT_PATH_SPRING_PROPERTY = "grok.path.default";
public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp";
public static final String GROK_CLASS_NAME = GrokParser.class.getName();
public static final String GROK_PATH_KEY = "grokPath";
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index adeb1ed..807a330 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -17,10 +17,10 @@
*/
package org.apache.metron.rest.service;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
-import java.io.File;
import java.util.Map;
public interface GrokService {
@@ -29,7 +29,9 @@ public interface GrokService {
GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
- File saveTemporary(String statement, String name) throws RestException;
+ Path saveTemporary(String statement, String name) throws RestException;
+
+ void deleteTemporary() throws RestException;
String getStatementFromClasspath(String path) throws RestException;
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index d5932c7..58dbf9b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -31,4 +31,6 @@ public interface HdfsService {
List<String> list(Path path) throws RestException;
boolean delete(Path path, boolean recursive) throws RestException;
+
+ boolean mkdirs(Path path) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index edae13b..e185b53 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service.impl;
+import java.nio.charset.Charset;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.commons.io.IOUtils;
@@ -25,15 +26,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.HdfsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Map;
@@ -47,10 +46,13 @@ public class GrokServiceImpl implements GrokService {
private Grok commonGrok;
+ private HdfsService hdfsService;
+
@Autowired
- public GrokServiceImpl(Environment environment, Grok commonGrok) {
+ public GrokServiceImpl(Environment environment, Grok commonGrok, HdfsService hdfsService) {
this.environment = environment;
this.commonGrok = commonGrok;
+ this.hdfsService = hdfsService;
}
@Override
@@ -85,30 +87,25 @@ public class GrokServiceImpl implements GrokService {
}
@Override
- public File saveTemporary(String statement, String name) throws RestException {
+ public Path saveTemporary(String statement, String name) throws RestException {
if (statement != null) {
- try {
- File grokDirectory = new File(getTemporaryGrokRootPath());
- if (!grokDirectory.exists()) {
- grokDirectory.mkdirs();
- }
- File path = new File(grokDirectory, name);
- FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
- fileWriter.write(statement);
- fileWriter.close();
- return path;
- } catch (IOException e) {
- throw new RestException(e);
- }
+ Path path = getTemporaryGrokRootPath();
+ hdfsService.mkdirs(path);
+ hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")));
+ return path;
} else {
throw new RestException("A grokStatement must be provided");
}
}
- private String getTemporaryGrokRootPath() {
+ public void deleteTemporary() throws RestException {
+ hdfsService.delete(getTemporaryGrokRootPath(), true);
+ }
+
+ private Path getTemporaryGrokRootPath() {
String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
- return new Path(grokTempPath, authentication.getName()).toString();
+ return new Path(grokTempPath, authentication.getName());
}
public String getStatementFromClasspath(String path) throws RestException {
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index 789c421..a9ae8eb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -88,4 +88,13 @@ public class HdfsServiceImpl implements HdfsService {
throw new RestException(e);
}
}
+
+ @Override
+ public boolean mkdirs(Path path) throws RestException {
+ try {
+ return FileSystem.get(configuration).mkdirs(path);
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 37d59d0..f99b41c 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,8 +17,16 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
+
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
@@ -34,15 +42,6 @@ import org.reflections.Reflections;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-
@Service
public class SensorParserConfigServiceImpl implements SensorParserConfigService {
@@ -53,7 +52,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
private GrokService grokService;
@Autowired
- public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) {
+ public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client,
+ GrokService grokService) {
this.objectMapper = objectMapper;
this.client = client;
this.grokService = grokService;
@@ -64,7 +64,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
@Override
public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
try {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(),
+ objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
} catch (Exception e) {
throw new RestException(e);
}
@@ -126,7 +127,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
Set<Class<? extends MessageParser>> parserClasses = getParserClasses();
parserClasses.forEach(parserClass -> {
if (!"BasicParser".equals(parserClass.getSimpleName())) {
- availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), parserClass.getName());
+ availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""),
+ parserClass.getName());
}
});
}
@@ -154,20 +156,23 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else {
MessageParser<JSONObject> parser;
try {
- parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
+ parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName())
+ .newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
- File temporaryGrokFile = null;
+ Path temporaryGrokPath = null;
if (isGrokConfig(sensorParserConfig)) {
- temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic());
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString());
+ String name = parseMessageRequest.getSensorParserConfig().getSensorTopic();
+ temporaryGrokPath = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), name);
+ sensorParserConfig.getParserConfig()
+ .put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString());
}
parser.configure(sensorParserConfig.getParserConfig());
parser.init();
JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
- if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) {
- temporaryGrokFile.delete();
+ if (isGrokConfig(sensorParserConfig) && temporaryGrokPath != null) {
+ grokService.deleteTemporary();
}
return results;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
index 15fa293..426a0da 100644
--- a/metron-interface/metron-rest/src/main/resources/application-docker.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -46,7 +46,6 @@ hdfs:
grok:
path:
temp: target/patterns/temp
- default: target/patterns
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-test.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml
index 9793840..b5e65a7 100644
--- a/metron-interface/metron-rest/src/main/resources/application-test.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -29,7 +29,6 @@ spring:
grok:
path:
temp: target/patterns/temp
- default: target/patterns
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
index 31b5784..cf2c170 100644
--- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -39,8 +39,7 @@ hdfs:
grok:
path:
- temp: ./patterns/temp
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index cf9218b..d96439c 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -26,8 +26,7 @@ spring:
grok:
path:
- temp: ./
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
zookeeper:
client:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index 66771eb..6e2d788 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -21,7 +21,6 @@ import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.rest.MetronRestConstants;
-import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.junit.Before;
import org.junit.Test;
@@ -365,11 +364,6 @@ public class SensorParserConfigControllerIntegrationTest {
FileUtils.cleanDirectory(grokTempPath);
FileUtils.deleteDirectory(grokTempPath);
}
- File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY));
- if (grokPath.exists()) {
- FileUtils.cleanDirectory(grokPath);
- FileUtils.deleteDirectory(grokPath);
- }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index 1935269..8f7f40c 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -17,51 +17,46 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
import oi.thekraken.grok.api.Grok;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.HdfsService;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.core.env.Environment;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({GrokServiceImpl.class, FileWriter.class})
public class GrokServiceImplTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
private Environment environment;
private Grok grok;
+ private HdfsService hdfsService;
private GrokService grokService;
@Before
public void setUp() throws Exception {
environment = mock(Environment.class);
grok = mock(Grok.class);
- grokService = new GrokServiceImpl(environment, grok);
+ hdfsService = new HdfsServiceImpl(new Configuration());
+ grokService = new GrokServiceImpl(environment, grok, hdfsService);
}
@Test
@@ -205,24 +200,9 @@ public class GrokServiceImplTest {
grokService.saveTemporary(statement, "squid");
- File testFile = new File("./target/user1/squid");
- assertEquals(statement, FileUtils.readFileToString(testFile));
- testFile.delete();
- }
-
- @Test
- public void saveTemporaryShouldWrapExceptionInRestException() throws Exception {
- exception.expect(RestException.class);
-
- String statement = "grok statement";
-
- Authentication authentication = mock(Authentication.class);
- when(authentication.getName()).thenReturn("user1");
- SecurityContextHolder.getContext().setAuthentication(authentication);
- when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
- whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException());
-
- grokService.saveTemporary(statement, "squid");
+ File testRoot = new File("./target/user1");
+ assertEquals(statement, FileUtils.readFileToString(new File(testRoot, "squid"), Charset.forName("utf-8")));
+ testRoot.delete();
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
index d35a48c..c96a796 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -18,12 +18,14 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
import org.apache.curator.framework.api.GetChildrenBuilder;
import org.apache.curator.framework.api.GetDataBuilder;
import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.rest.RestException;
@@ -43,7 +45,10 @@ import java.io.FileWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -88,11 +93,18 @@ public class SensorParserConfigServiceImplTest {
@Multiline
public static String broJson;
+ private String user = "user1";
+
@Before
public void setUp() throws Exception {
objectMapper = mock(ObjectMapper.class);
curatorFramework = mock(CuratorFramework.class);
- grokService = mock(GrokService.class);
+ Environment environment = mock(Environment.class);
+ Authentication authentication = mock(Authentication.class);
+ when(authentication.getName()).thenReturn(user);
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+ grokService = new GrokServiceImpl(environment, mock(Grok.class), new HdfsServiceImpl(new Configuration()));
sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService);
}
@@ -269,13 +281,13 @@ public class SensorParserConfigServiceImplTest {
parseMessageRequest.setGrokStatement(grokStatement);
parseMessageRequest.setSampleData(sampleData);
- File patternFile = new File("./target/squidTest");
+ File grokRoot = new File("./target", user);
+ grokRoot.mkdir();
+ File patternFile = new File(grokRoot, "squid");
FileWriter writer = new FileWriter(patternFile);
writer.write(grokStatement);
writer.close();
- when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile);
-
assertEquals(new HashMap() {{
put("elapsed", 161);
put("code", 200);