You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/03/02 20:51:47 UTC
[01/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Repository: incubator-metron
Updated Branches:
refs/heads/master e416a7d78 -> e66284993
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
new file mode 100644
index 0000000..dd21095
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.mock;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.service.impl.StormCLIWrapper;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class MockStormCLIClientWrapper extends StormCLIWrapper {
+
+ private final Map<String, TopologyStatusCode> parsersStatus = new HashMap<>();
+ private TopologyStatusCode enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ private TopologyStatusCode indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+
+ public Set<String> getParserTopologyNames() {
+ return parsersStatus.keySet();
+ }
+
+ public TopologyStatusCode getParserStatus(String name) {
+ TopologyStatusCode parserStatus = parsersStatus.get(name);
+ if (parserStatus == null) {
+ return TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ } else {
+ return parserStatus;
+ }
+ }
+
+ @Override
+ public int startParserTopology(String name) throws RestException {
+ TopologyStatusCode parserStatus = parsersStatus.get(name);
+ if (parserStatus == null || parserStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ parsersStatus.put(name, TopologyStatusCode.ACTIVE);
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override
+ public int stopParserTopology(String name, boolean stopNow) throws RestException {
+ TopologyStatusCode parserStatus = parsersStatus.get(name);
+ if (parserStatus == TopologyStatusCode.ACTIVE) {
+ parsersStatus.put(name, TopologyStatusCode.TOPOLOGY_NOT_FOUND);
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int activateParserTopology(String name) {
+ TopologyStatusCode parserStatus = parsersStatus.get(name);
+ if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) {
+ parsersStatus.put(name, TopologyStatusCode.ACTIVE);
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int deactivateParserTopology(String name) {
+ TopologyStatusCode parserStatus = parsersStatus.get(name);
+ if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) {
+ parsersStatus.put(name, TopologyStatusCode.INACTIVE);
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public TopologyStatusCode getEnrichmentStatus() {
+ return enrichmentStatus;
+ }
+
+ @Override
+ public int startEnrichmentTopology() throws RestException {
+ if (enrichmentStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ enrichmentStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override
+ public int stopEnrichmentTopology(boolean stopNow) throws RestException {
+ if (enrichmentStatus == TopologyStatusCode.ACTIVE) {
+ enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int activateEnrichmentTopology() {
+ if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) {
+ enrichmentStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int deactivateEnrichmentTopology() {
+ if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) {
+ enrichmentStatus = TopologyStatusCode.INACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public TopologyStatusCode getIndexingStatus() {
+ return indexingStatus;
+ }
+
+ @Override
+ public int startIndexingTopology() throws RestException {
+ if (indexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ indexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override
+ public int stopIndexingTopology(boolean stopNow) throws RestException {
+ if (indexingStatus == TopologyStatusCode.ACTIVE) {
+ indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int activateIndexingTopology() {
+ if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) {
+ indexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public int deactivateIndexingTopology() {
+ if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) {
+ indexingStatus = TopologyStatusCode.INACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ @Override
+ protected String stormClientVersionInstalled() throws RestException {
+ return "1.0.1";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
new file mode 100644
index 0000000..ca5c1c9
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.mock;
+
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.model.TopologyStatus;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.model.TopologySummary;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MockStormRestTemplate extends RestTemplate {
+
+ @Autowired
+ private Environment environment;
+
+ private MockStormCLIClientWrapper mockStormCLIClientWrapper;
+
+ public void setMockStormCLIClientWrapper(MockStormCLIClientWrapper mockStormCLIClientWrapper) {
+ this.mockStormCLIClientWrapper = mockStormCLIClientWrapper;
+ }
+
+ @Override
+ public Object getForObject(String url, Class responseType, Object... urlVariables) throws RestClientException {
+ Object response = null;
+ if (url.equals("http://" + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_SUMMARY_URL)) {
+ TopologySummary topologySummary = new TopologySummary();
+ List<TopologyStatus> topologyStatusList = new ArrayList<>();
+ for(String name: mockStormCLIClientWrapper.getParserTopologyNames()) {
+ topologyStatusList.add(getTopologyStatus(name));
+ }
+ TopologyStatusCode enrichmentStatus = mockStormCLIClientWrapper.getEnrichmentStatus();
+ if (enrichmentStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ topologyStatusList.add(getTopologyStatus("enrichment"));
+ }
+ TopologyStatusCode indexingStatus = mockStormCLIClientWrapper.getIndexingStatus();
+ if (indexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ topologyStatusList.add(getTopologyStatus("indexing"));
+ }
+ topologySummary.setTopologies(topologyStatusList.toArray(new TopologyStatus[topologyStatusList.size()]));
+ response = topologySummary;
+ } else if (url.startsWith("http://" + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_URL + "/")){
+ String name = url.substring(url.lastIndexOf('/') + 1, url.length()).replaceFirst("-id", "");
+ response = getTopologyStatus(name);
+ }
+ return response;
+ }
+
+ private TopologyStatus getTopologyStatus(String name) {
+ TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setName(name);
+ topologyStatus.setId(name + "-id");
+ if ("enrichment".equals(name)) {
+ topologyStatus.setStatus(mockStormCLIClientWrapper.getEnrichmentStatus());
+ } else if ("indexing".equals(name)) {
+ topologyStatus.setStatus(mockStormCLIClientWrapper.getIndexingStatus());
+ } else {
+ topologyStatus.setStatus(mockStormCLIClientWrapper.getParserStatus(name));
+ }
+ return topologyStatus;
+ }
+
+ @Override
+ public Object postForObject(String url, Object request, Class responseType, Object... uriVariables) throws RestClientException {
+ Map<String, String> result = new HashMap<>();
+ String[] urlParts = url.split("/");
+ String name = urlParts[urlParts.length - 2].replaceFirst("-id", "");
+ String action = urlParts[urlParts.length - 1];
+ int returnCode = 0;
+ if (action.equals("activate")) {
+ if (name.equals("enrichment")) {
+ returnCode = mockStormCLIClientWrapper.activateEnrichmentTopology();
+ } else if (name.equals("indexing")) {
+ returnCode = mockStormCLIClientWrapper.activateIndexingTopology();
+ } else {
+ returnCode = mockStormCLIClientWrapper.activateParserTopology(name);
+ }
+ } else if (action.equals("deactivate")){
+ if (name.equals("enrichment")) {
+ returnCode = mockStormCLIClientWrapper.deactivateEnrichmentTopology();
+ } else if (name.equals("indexing")) {
+ returnCode = mockStormCLIClientWrapper.deactivateIndexingTopology();
+ } else {
+ returnCode = mockStormCLIClientWrapper.deactivateParserTopology(name);
+ }
+ }
+ if (returnCode == 0) {
+ result.put("status", "success");
+ } else {
+ result.put("status", "error");
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
new file mode 100644
index 0000000..f7e43ab
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.config.HadoopConfig;
+import org.apache.metron.rest.service.impl.HdfsServiceImpl;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceTest.HdfsServiceTestContextConfiguration.class})
+@ActiveProfiles(TEST_PROFILE)
+public class HdfsServiceTest {
+
+ @Configuration
+ @Profile("test")
+ static class HdfsServiceTestContextConfiguration {
+
+ @Bean
+ public HdfsService hdfsService() {
+ return new HdfsServiceImpl();
+ }
+ }
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ @Test
+ public void test() throws IOException {
+ String rootDir = "./src/test/tmp";
+ File rootFile = new File(rootDir);
+ Path rootPath = new Path(rootDir);
+ if (rootFile.exists()) {
+ FileUtils.cleanDirectory(rootFile);
+ FileUtils.deleteDirectory(rootFile);
+ }
+ assertEquals(true, rootFile.mkdir());
+ String fileName1 = "fileName1";
+ String fileName2 = "fileName2";
+ Path path1 = new Path(rootDir, fileName1);
+ String value1 = "value1";
+ String value2 = "value2";
+ Path path2 = new Path(rootDir, fileName2);
+ String invalidFile = "invalidFile";
+ Path pathInvalidFile = new Path(rootDir, invalidFile);
+
+ FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir));
+ assertEquals(0, fileStatuses.length);
+
+
+ hdfsService.write(path1, value1.getBytes());
+ assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1)));
+ assertEquals(value1, new String(hdfsService.read(path1)));
+
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(1, fileStatuses.length);
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+
+ hdfsService.write(path2, value2.getBytes());
+ assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2)));
+ assertEquals(value2, new String(hdfsService.read(path2)));
+
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(2, fileStatuses.length);
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+
+ assertEquals(true, hdfsService.delete(path1, false));
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(1, fileStatuses.length);
+ assertEquals(fileName2, fileStatuses[0].getPath().getName());
+ assertEquals(true, hdfsService.delete(path2, false));
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(0, fileStatuses.length);
+
+ try {
+ hdfsService.read(pathInvalidFile);
+ fail("Exception should be thrown when reading invalid file name");
+ } catch(IOException e) {
+ }
+ assertEquals(false, hdfsService.delete(pathInvalidFile, false));
+
+ FileUtils.deleteDirectory(new File(rootDir));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
new file mode 100644
index 0000000..561548e
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.GetChildrenBuilder;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.rest.service.impl.SensorParserConfigServiceImpl;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.verifyStatic;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ConfigurationsUtils.class})
+public class SensorParserConfigTest {
+
+ @Mock
+ private GetChildrenBuilder getChildrenBuilder;
+
+ @Mock
+ private DeleteBuilder deleteBuilder;
+
+ @Mock
+ private ObjectMapper objectMapper;
+
+ @Mock
+ private CuratorFramework client;
+
+ @Mock
+ private GrokService grokService;
+
+ @InjectMocks
+ private SensorParserConfigServiceImpl sensorParserConfigService;
+
+ @Before
+ public void setUp() throws Exception {
+ MockitoAnnotations.initMocks(this);
+ Mockito.when(client.getChildren()).thenReturn(getChildrenBuilder);
+ Mockito.when(client.delete()).thenReturn(deleteBuilder);
+
+ }
+
+ @Test
+ public void test() throws Exception {
+ mockStatic(ConfigurationsUtils.class);
+ SensorParserConfig broParserConfig = new SensorParserConfig();
+ broParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
+ broParserConfig.setSensorTopic("broTest");
+ Mockito.when(objectMapper.writeValueAsString(broParserConfig)).thenReturn(new String(JSONUtils.INSTANCE.toJSON(broParserConfig)));
+ sensorParserConfigService.save(broParserConfig);
+ verifyStatic(times(1));
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper("broTest", JSONUtils.INSTANCE.toJSON(broParserConfig), client);
+
+ PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("broTest", client)).thenReturn(broParserConfig);
+ assertEquals(broParserConfig, sensorParserConfigService.findOne("broTest"));
+
+ SensorParserConfig squidParserConfig = new SensorParserConfig();
+ squidParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser");
+ squidParserConfig.setSensorTopic("squid");
+ PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("squidTest", client)).thenReturn(squidParserConfig);
+
+ List<String> allTypes = new ArrayList<String>() {{
+ add("broTest");
+ add("squidTest");
+ }};
+ Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenReturn(allTypes);
+ assertEquals(new ArrayList<SensorParserConfig>() {{ add(broParserConfig); add(squidParserConfig); }}, sensorParserConfigService.getAll());
+
+ Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(new KeeperException.NoNodeException());
+ assertEquals(new ArrayList<>(), sensorParserConfigService.getAll());
+
+ assertTrue(sensorParserConfigService.delete("broTest"));
+ verify(deleteBuilder, times(1)).forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest");
+ Mockito.when(deleteBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest")).thenThrow(new KeeperException.NoNodeException());
+ assertFalse(sensorParserConfigService.delete("broTest"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java
new file mode 100644
index 0000000..d9522f2
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.utils;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.reflections.Reflections;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.FileWriter;
+import java.lang.reflect.Method;
+import java.lang.reflect.Parameter;
+import java.security.Principal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+public class ReadMeUtils {
+
+ public static void main(String[] args) throws Exception {
+ String path;
+ if (args.length == 0) {
+ System.out.println("README output path was not set. Defaulting to 'metron-interface/metron-rest/README.md'");
+ path = "metron-interface/metron-rest/README.md";
+ } else {
+ path = args[0];
+ }
+
+ Reflections reflections = new Reflections("org.apache.metron.rest.controller");
+ List<RestControllerInfo> endpoints = new ArrayList<>();
+ Set<Class<?>> controllers = reflections.getTypesAnnotatedWith(RestController.class);
+ for(Class<?> controller: controllers) {
+ RequestMapping controllerRequestMapping = controller.getAnnotation(RequestMapping.class);
+ String pathPrefix;
+ if (controllerRequestMapping == null) {
+ pathPrefix = "";
+ } else {
+ pathPrefix = controllerRequestMapping.value()[0];
+ }
+ for(Method method: controller.getDeclaredMethods()) {
+ RestControllerInfo restControllerInfo = new RestControllerInfo();
+ RequestMapping requestMapping = method.getAnnotation(RequestMapping.class);
+ if (requestMapping == null) {
+ throw new Exception(String.format("@RequestMapping annotation missing from method %s.%s. Every controller method must have a @RequestMapping annotation.", controller.getSimpleName(), method.getName()));
+ }
+ ApiOperation apiOperation = method.getAnnotation(ApiOperation.class);
+ if (apiOperation == null) {
+ throw new Exception(String.format("@ApiOperation annotation missing from method %s.%s. Every controller method must have an @ApiOperation annotation.", controller.getSimpleName(), method.getName()));
+ }
+ ApiResponse[] apiResponseArray;
+ ApiResponses apiResponses = method.getAnnotation(ApiResponses.class);
+ if (apiResponses == null) {
+ ApiResponse apiResponse = method.getAnnotation(ApiResponse.class);
+ if (apiResponse == null) {
+ throw new Exception(String.format("@ApiResponses or @ApiResponse annotation missing from method %s.%s. Every controller method must have an @ApiResponses or @ApiResponse annotation.", controller.getSimpleName(), method.getName()));
+ } else {
+ apiResponseArray = new ApiResponse[]{ apiResponse };
+ }
+ } else {
+ apiResponseArray = apiResponses.value();
+ }
+ String[] requestMappingValue = requestMapping.value();
+ if (requestMappingValue.length == 0) {
+ restControllerInfo.setPath(pathPrefix);
+ } else {
+ restControllerInfo.setPath(pathPrefix + requestMappingValue[0]);
+ }
+ restControllerInfo.setDescription(apiOperation.value());
+ RequestMethod requestMethod;
+ if (requestMapping.method().length == 0) {
+ requestMethod = RequestMethod.GET;
+ } else {
+ requestMethod = requestMapping.method()[0];
+ }
+ restControllerInfo.setMethod(requestMethod);
+ for (ApiResponse apiResponse: apiResponseArray) {
+ restControllerInfo.addResponse(apiResponse.message(), apiResponse.code());
+ }
+ for(Parameter parameter: method.getParameters()) {
+ if (!parameter.getType().equals(Principal.class)) {
+ ApiParam apiParam = parameter.getAnnotation(ApiParam.class);
+ if (apiParam == null) {
+ throw new Exception(String.format("@ApiParam annotation missing from parameter %s.%s.%s. Every controller method parameter must have an @ApiParam annotation.", controller.getSimpleName(), method.getName(), parameter.getName()));
+ }
+ restControllerInfo.addParameterDescription(apiParam.name(), apiParam.value());
+ }
+ }
+ endpoints.add(restControllerInfo);
+ }
+ }
+ Collections.sort(endpoints, (o1, o2) -> o1.getPath().compareTo(o2.getPath()));
+
+ Velocity.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");
+ Velocity.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
+ Velocity.init();
+
+ VelocityContext context = new VelocityContext();
+ context.put( "endpoints", endpoints );
+
+ Template template = Velocity.getTemplate("README.vm");
+ FileWriter fileWriter = new FileWriter(path);
+ template.merge( context, fileWriter );
+ fileWriter.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java
new file mode 100644
index 0000000..5b7ac0e
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.utils;
+
+import org.springframework.web.bind.annotation.RequestMethod;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RestControllerInfo {
+
+ public class Response {
+
+ private String message;
+ private int code;
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public void setCode(int code) {
+ this.code = code;
+ }
+ }
+
+ private String path;
+ private String description;
+ private RequestMethod method;
+ private List<Response> responses = new ArrayList<>();
+ private Map<String, String> parameterDescriptions = new HashMap<>();
+
+ public RestControllerInfo() {}
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public RequestMethod getMethod() {
+ return method;
+ }
+
+ public void setMethod(RequestMethod method) {
+ this.method = method;
+ }
+
+ public List<Response> getResponses() {
+ return responses;
+ }
+
+ public void addResponse(String message, int code) {
+ Response response = new Response();
+ response.setMessage(message);
+ response.setCode(code);
+ this.responses.add(response);
+ }
+
+ public Map<String, String> getParameterDescriptions() {
+ return parameterDescriptions;
+ }
+
+ public void addParameterDescription(String name, String description) {
+ parameterDescriptions.put(name, description);
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/resources/README.vm
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/resources/README.vm b/metron-interface/metron-rest/src/test/resources/README.vm
new file mode 100644
index 0000000..f259a91
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/resources/README.vm
@@ -0,0 +1,92 @@
+#[[#]]# Metron REST and Configuration UI
+
+This UI exposes and aids in sensor configuration.
+
+#[[##]]# Prerequisites
+
+* A running Metron cluster
+* A running instance of MySQL
+* Java 8 installed
+* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
+
+#[[##]]# Installation
+1. Package the Application with Maven:
+ ```
+ mvn clean package
+ ```
+
+1. Untar the archive in the target directory. The directory structure will look like:
+ ```
+ bin
+ start.sh
+ lib
+ metron-rest-version.jar
+ ```
+
+1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
+ ```
+ export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
+ ```
+
+1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
+ ```
+ export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
+ ```
+
+1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+
+1. Create a Config UI database in MySQL with this command:
+ ```
+ CREATE DATABASE IF NOT EXISTS metronrest
+ ```
+
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+
+1. Start the UI with this command:
+ ```
+ ./bin/start.sh /path/to/application.yml
+ ```
+
+#[[##]]# Usage
+
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement:
+```
+use metronrest;
+insert into users (username, password, enabled) values ('your_username','your_password',1);
+insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
+```
+Users can be added to additional groups with this SQL statement:
+```
+use metronrest;
+insert into authorities (username, authority) values ('your_username', 'your_group');
+```
+
+#[[##]]# API
+
+Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI.
+
+| |
+| ---------- |
+#foreach( $restControllerInfo in $endpoints )
+| [ `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()`](#$restControllerInfo.getMethod().toString().toLowerCase()-$restControllerInfo.getPath().toLowerCase().replaceAll("/", ""))|
+#end
+
+#foreach( $restControllerInfo in $endpoints )
+#[[###]]# `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()`
+ * Description: $restControllerInfo.getDescription()
+#if($restControllerInfo.getParameterDescriptions().size() > 0)
+ * Input:
+#end
+#foreach( $parameterDescription in $restControllerInfo.getParameterDescriptions().entrySet())
+ * $parameterDescription.getKey() - $parameterDescription.getValue()
+#end
+ * Returns:
+#foreach( $response in $restControllerInfo.getResponses())
+ * $response.getCode() - $response.getMessage()
+#end
+
+#end
+
+#[[##]]# License
+
+This project depends on the Java Transaction API. See https://java.net/projects/jta-spec/ for more details.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/resources/log4j.properties b/metron-interface/metron-rest/src/test/resources/log4j.properties
new file mode 100644
index 0000000..492cecf
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/resources/log4j.properties
@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=ERROR, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/pom.xml b/metron-interface/pom.xml
new file mode 100644
index 0000000..078da19
--- /dev/null
+++ b/metron-interface/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-interface</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-interface</name>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>Metron</artifactId>
+ <version>0.3.0</version>
+ </parent>
+ <description>Interfaces for Metron</description>
+ <url>https://metron.incubator.apache.org/</url>
+ <scm>
+ <connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git</connection>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://git-wip-us.apache.org/repos/asf/incubator-metron</url>
+ </scm>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-rest</module>
+ <module>metron-rest-client</module>
+ </modules>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.adrianwalker</groupId>
+ <artifactId>multiline-string</artifactId>
+ <version>0.1.2</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ </plugins>
+ </build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <targetJdk>${global_java_version}</targetJdk>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ <inherited>true</inherited>
+ </plugin>
+ </plugins>
+ </reporting>
+ <repositories>
+ <repository>
+ <id>multiline-release-repo</id>
+ <url>https://raw.github.com/benelog/multiline/master/maven-repository</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index db28adb..3a8eb9b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,6 +29,7 @@
<module>metron-platform</module>
<module>metron-deployment</module>
<module>metron-docker</module>
+ <module>metron-interface</module>
</modules>
<repositories>
[06/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
METRON-503: Metron REST API this closes apache/incubator-metron#316
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/55b3e7ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/55b3e7ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/55b3e7ea
Branch: refs/heads/master
Commit: 55b3e7ea80953a3bbfad5ec43b85d17e8ed287e6
Parents: e416a7d
Author: merrimanr <me...@gmail.com>
Authored: Thu Mar 2 10:48:00 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Mar 2 10:48:00 2017 -0500
----------------------------------------------------------------------
dependencies_with_url.csv | 60 ++
metron-interface/metron-rest-client/pom.xml | 40 ++
.../metron/rest/model/GrokValidation.java | 55 ++
.../apache/metron/rest/model/KafkaTopic.java | 60 ++
.../metron/rest/model/ParseMessageRequest.java | 42 ++
.../org/apache/metron/rest/model/RestError.java | 43 ++
.../rest/model/StellarFunctionDescription.java | 52 ++
.../metron/rest/model/TopologyResponse.java | 42 ++
.../metron/rest/model/TopologyResponseCode.java | 23 +
.../metron/rest/model/TopologyStatus.java | 76 ++
.../metron/rest/model/TopologyStatusCode.java | 32 +
.../metron/rest/model/TopologySummary.java | 34 +
.../rest/model/TransformationValidation.java | 51 ++
metron-interface/metron-rest/README.md | 439 ++++++++++++
metron-interface/metron-rest/pom.xml | 328 +++++++++
.../metron-rest/src/main/assembly/assembly.xml | 54 ++
.../metron/rest/MetronRestApplication.java | 29 +
.../apache/metron/rest/MetronRestConstants.java | 55 ++
.../org/apache/metron/rest/RestException.java | 43 ++
.../apache/metron/rest/config/GrokConfig.java | 36 +
.../apache/metron/rest/config/HadoopConfig.java | 38 +
.../apache/metron/rest/config/KafkaConfig.java | 61 ++
.../apache/metron/rest/config/MvcConfig.java | 31 +
.../metron/rest/config/RestTemplateConfig.java | 36 +
.../apache/metron/rest/config/StormConfig.java | 48 ++
.../metron/rest/config/SwaggerConfig.java | 39 ++
.../metron/rest/config/WebSecurityConfig.java | 93 +++
.../metron/rest/config/ZookeeperConfig.java | 52 ++
.../rest/controller/GlobalConfigController.java | 79 +++
.../metron/rest/controller/GrokController.java | 56 ++
.../metron/rest/controller/KafkaController.java | 96 +++
.../rest/controller/RestExceptionHandler.java | 48 ++
.../SensorEnrichmentConfigController.java | 97 +++
.../SensorIndexingConfigController.java | 88 +++
.../SensorParserConfigController.java | 112 +++
.../metron/rest/controller/StormController.java | 190 +++++
.../controller/TransformationController.java | 80 +++
.../metron/rest/controller/UserController.java | 36 +
.../rest/service/GlobalConfigService.java | 31 +
.../apache/metron/rest/service/GrokService.java | 31 +
.../apache/metron/rest/service/HdfsService.java | 35 +
.../metron/rest/service/KafkaService.java | 37 +
.../service/SensorEnrichmentConfigService.java | 40 ++
.../service/SensorIndexingConfigService.java | 37 +
.../rest/service/SensorParserConfigService.java | 42 ++
.../metron/rest/service/StormAdminService.java | 40 ++
.../metron/rest/service/StormStatusService.java | 37 +
.../rest/service/TransformationService.java | 39 ++
.../service/impl/DockerStormCLIWrapper.java | 70 ++
.../service/impl/GlobalConfigServiceImpl.java | 75 ++
.../rest/service/impl/GrokServiceImpl.java | 66 ++
.../rest/service/impl/HdfsServiceImpl.java | 62 ++
.../rest/service/impl/KafkaServiceImpl.java | 124 ++++
.../impl/SensorEnrichmentConfigServiceImpl.java | 112 +++
.../impl/SensorIndexingConfigServiceImpl.java | 106 +++
.../impl/SensorParserConfigServiceImpl.java | 289 ++++++++
.../service/impl/StormAdminServiceImpl.java | 100 +++
.../rest/service/impl/StormCLIWrapper.java | 150 ++++
.../service/impl/StormStatusServiceImpl.java | 122 ++++
.../service/impl/TransformationServiceImpl.java | 92 +++
.../src/main/resources/META-INF/LICENSE | 692 +++++++++++++++++++
.../src/main/resources/META-INF/NOTICE | 6 +
.../src/main/resources/application-docker.yml | 57 ++
.../src/main/resources/application-test.yml | 42 ++
.../src/main/resources/application.yml | 42 ++
.../src/main/resources/log4j.properties | 16 +
.../src/main/resources/schema-h2.sql | 30 +
.../src/main/resources/schema-mysql.sql | 30 +
.../metron-rest/src/main/scripts/start.sh | 33 +
.../metron/rest/MetronRestApplicationTest.java | 42 ++
.../apache/metron/rest/config/TestConfig.java | 124 ++++
.../GlobalConfigControllerIntegrationTest.java | 107 +++
.../GrokControllerIntegrationTest.java | 115 +++
.../KafkaControllerIntegrationTest.java | 188 +++++
...richmentConfigControllerIntegrationTest.java | 240 +++++++
...IndexingConfigControllerIntegrationTest.java | 140 ++++
...orParserConfigControllerIntegrationTest.java | 368 ++++++++++
.../StormControllerIntegrationTest.java | 320 +++++++++
...TransformationControllerIntegrationTest.java | 122 ++++
.../UserControllerIntegrationTest.java | 65 ++
.../rest/generator/SampleDataGenerator.java | 170 +++++
.../rest/mock/MockStormCLIClientWrapper.java | 177 +++++
.../metron/rest/mock/MockStormRestTemplate.java | 116 ++++
.../metron/rest/service/HdfsServiceTest.java | 117 ++++
.../rest/service/SensorParserConfigTest.java | 116 ++++
.../apache/metron/rest/utils/ReadMeUtils.java | 130 ++++
.../metron/rest/utils/RestControllerInfo.java | 103 +++
.../metron-rest/src/test/resources/README.vm | 92 +++
.../src/test/resources/log4j.properties | 16 +
metron-interface/pom.xml | 90 +++
pom.xml | 1 +
91 files changed, 8518 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index fded057..35acced 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -7,6 +7,7 @@ org.ow2.asm:asm:jar:5.0.3:compile,BSD,http://asm.ow2.org/
org.reflections:reflections:jar:0.9.10:compile,BSD,https://github.com/ronmamo/reflections
org.javassist:javassist:jar:3.19.0-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist
org.javassist:javassist:jar:3.17.1-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist
+org.javassist:javassist:jar:3.20.0-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist
org.fusesource.jansi:jansi:jar:1.11:compile,Apache v2,https://github.com/fusesource/jansi
de.javakaffee:kryo-serializers:jar:0.38:compile,Apache v2,https://github.com/magro/kryo-serializers
com.tdunning:t-digest:jar:3.1:compile,Apache v2,https://github.com/tdunning/t-digest
@@ -78,6 +79,7 @@ org.slf4j:slf4j-api:jar:1.7.10:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-api:jar:1.7.5:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-api:jar:1.7.6:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-api:jar:1.7.7:compile,MIT,http://www.slf4j.org
+org.slf4j:slf4j-api:jar:1.7.21:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.6.1:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.7.10:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.7.10:runtime,MIT,http://www.slf4j.org
@@ -85,6 +87,8 @@ org.slf4j:slf4j-log4j12:jar:1.7.21:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.7.5:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.7.7:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
aopalliance:aopalliance:jar:1.0:compile,Public Domain,http://aopalliance.sourceforge.net
com.101tec:zkclient:jar:0.8:compile,The Apache Software License, Version 2.0,https://github.com/sgroschupf/zkclient
com.github.stephenc.findbugs:findbugs-annotations:jar:1.3.9-1:compile,Apache License, Version 2.0,http://stephenc.github.com/findbugs-annotations
@@ -98,15 +102,21 @@ com.codahale.metrics:metrics-graphite:jar:3.0.2:compile,MIT,https://github.com/c
com.esotericsoftware.reflectasm:reflectasm:jar:shaded:1.07:compile,BSD,https://github.com/EsotericSoftware/reflectasm
com.fasterxml.jackson.core:jackson-annotations:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
com.fasterxml.jackson.core:jackson-annotations:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-annotations:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
com.fasterxml.jackson.core:jackson-core:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson-core
com.fasterxml.jackson.core:jackson-core:jar:2.7.4:compile,ASLv2,https://github.com/FasterXML/jackson-core
+com.fasterxml.jackson.core:jackson-core:jar:2.8.3:compile,ASLv2,https://github.com/FasterXML/jackson-core
com.fasterxml.jackson.core:jackson-databind:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
com.fasterxml.jackson.core:jackson-databind:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-databind:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForCbor
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForSmile
com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson
+com.fasterxml.jackson.datatype:jackson-datatype-joda:jar:2.8.1:compile,ASLv2,https://github.com/FasterXML/jackson-datatype-joda
+com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
+com.google.code.gson:gson:jar:2.7:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
com.google.guava:guava:jar:11.0.2:compile,ASLv2,
com.google.guava:guava:jar:12.0.1:compile,ASLv2,
com.google.guava:guava:jar:12.0:compile,ASLv2,
@@ -129,6 +139,7 @@ commons-beanutils:commons-beanutils-core:jar:1.8.0:compile,ASLv2,http://commons.
commons-beanutils:commons-beanutils-core:jar:1.8.0:provided,ASLv2,http://commons.apache.org/beanutils/
commons-beanutils:commons-beanutils:jar:1.7.0:compile,ASLv2,
commons-beanutils:commons-beanutils:jar:1.8.3:compile,ASLv2,http://commons.apache.org/beanutils/
+commons-beanutils:commons-beanutils:jar:1.9.2:compile,ASLv2,http://commons.apache.org/beanutils/
commons-cli:commons-cli:jar:1.2:compile,ASLv2,http://commons.apache.org/cli/
commons-cli:commons-cli:jar:1.3.1:compile,ASLv2,http://commons.apache.org/proper/commons-cli/
commons-codec:commons-codec:jar:1.10:compile,ASLv2,http://commons.apache.org/proper/commons-codec/
@@ -142,12 +153,14 @@ commons-configuration:commons-configuration:jar:1.6:compile,The Apache Software
commons-daemon:commons-daemon:jar:1.0.13:compile,ASLv2,http://commons.apache.org/daemon/
commons-digester:commons-digester:jar:1.8.1:compile,ASLv2,http://commons.apache.org/digester/
commons-digester:commons-digester:jar:1.8:compile,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/digester/
+commons-digester:commons-digester:jar:2.1:compile,ASLv2,http://commons.apache.org/digester/
commons-el:commons-el:jar:1.0:provided,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/el/
commons-el:commons-el:jar:1.0:runtime,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/el/
commons-httpclient:commons-httpclient:jar:3.1:compile,Apache License,http://jakarta.apache.org/httpcomponents/httpclient-3.x/
commons-io:commons-io:jar:2.4:compile,ASLv2,http://commons.apache.org/io/
commons-io:commons-io:jar:2.5:compile,ASLv2,http://commons.apache.org/io/
commons-io:commons-io:jar:2.4:provided,ASLv2,http://commons.apache.org/io/
+commons-lang:commons-lang:jar:2.4:compile,ASLv2,http://commons.apache.org/lang/
commons-lang:commons-lang:jar:2.5:compile,ASLv2,http://commons.apache.org/lang/
commons-lang:commons-lang:jar:2.6:compile,ASLv2,http://commons.apache.org/lang/
commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/
@@ -204,14 +217,25 @@ org.springframework.integration:spring-integration-core:jar:3.0.0.RELEASE:compil
org.springframework.integration:spring-integration-http:jar:3.0.0.RELEASE:compile,The Apache Software License, Version 2.0,http://www.springintegration.org/
org.springframework.retry:spring-retry:jar:1.0.3.RELEASE:compile,Apache 2.0,http://www.springsource.org
org.springframework:spring-aop:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-aop:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework:spring-aspects:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-beans:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-beans:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-context:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-context:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-core:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
org.springframework:spring-core:jar:4.1.4.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework:spring-core:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-expression:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-expression:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework:spring-jdbc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework:spring-orm:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-tx:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-tx:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-web:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-web:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
+org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html
org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/
org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java
@@ -223,3 +247,39 @@ xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j
xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/
de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html
org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html
+xml-apis:xml-apis:jar:1.4.01:compile,ASLv2,http://xml.apache.org/commons/components/external/
+dom4j:dom4j:jar:1.6.1:compile,dom4j,https://github.com/dom4j/dom4j
+io.springfox:springfox-core:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-schema:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-spi:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-spring-web:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-swagger-common:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-swagger-ui:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.springfox:springfox-swagger2:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox
+io.swagger:swagger-annotations:jar:1.5.9:compile,ASLv2,https://github.com/swagger-api/swagger-core
+io.swagger:swagger-models:jar:1.5.9:compile,ASLv2,https://github.com/swagger-api/swagger-core
+javax.transaction:javax.transaction-api:jar:1.2:compile,CDDL-1.0,https://java.net/projects/jta-spec/
+javax.validation:validation-api:jar:1.1.0.Final:compile,ASLv2,http://beanvalidation.org
+joda-time:joda-time:jar:2.9.4:compile,ASLv2,https://github.com/JodaOrg/joda-time
+org.aspectj:aspectjweaver:jar:1.8.9:compile,EPL 1.0,https://eclipse.org/aspectj
+org.jboss.logging:jboss-logging:jar:3.3.0.Final:compile,ASLv2,https://github.com/jboss-logging
+org.jboss:jandex:jar:2.0.0.Final:compile,ASLv2,https://github.com/wildfly/jandex
+org.mapstruct:mapstruct:jar:1.0.0.Final:compile,ASLv2,https://github.com/mapstruct/mapstruct
+org.springframework.boot:spring-boot-autoconfigure:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-aop:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-data-jpa:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-jdbc:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-logging:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-security:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-tomcat:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter-web:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot-starter:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.boot:spring-boot:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot
+org.springframework.data:spring-data-commons:jar:1.12.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-data-commons
+org.springframework.data:spring-data-jpa:jar:1.10.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-data-jpa
+org.springframework.plugin:spring-plugin-core:jar:1.2.0.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-plugin
+org.springframework.plugin:spring-plugin-metadata:jar:1.2.0.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-plugin
+org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
+org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
+org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
+antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
new file mode 100644
index 0000000..66ccf52
--- /dev/null
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-interface</artifactId>
+ <version>0.3.0</version>
+ </parent>
+ <artifactId>metron-rest-client</artifactId>
+ <properties>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
new file mode 100644
index 0000000..7fbcd34
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class GrokValidation {
+
+ private String statement;
+ private String sampleData;
+ private Map<String, Object> results;
+
+ public String getStatement() {
+ return statement;
+ }
+
+ public void setStatement(String statement) {
+ this.statement = statement;
+ }
+
+ public String getSampleData() {
+ return sampleData;
+ }
+
+ public void setSampleData(String sampleData) {
+ this.sampleData = sampleData;
+ }
+
+ public Map<String, Object> getResults() {
+ if (results == null) {
+ return new HashMap<>();
+ }
+ return results;
+ }
+
+ public void setResults(Map<String, Object> results) {
+ this.results = results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
new file mode 100644
index 0000000..c8db9f6
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import java.util.Properties;
+
+public class KafkaTopic {
+
+ private String name;
+ private int numPartitions;
+ private int replicationFactor;
+ private Properties properties = new Properties();
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getNumPartitions() {
+ return numPartitions;
+ }
+
+ public void setNumPartitions(int numPartitions) {
+ this.numPartitions = numPartitions;
+ }
+
+ public int getReplicationFactor() {
+ return replicationFactor;
+ }
+
+ public void setReplicationFactor(int replicationFactor) {
+ this.replicationFactor = replicationFactor;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
new file mode 100644
index 0000000..8dfe17b
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+public class ParseMessageRequest {
+
+ private SensorParserConfig sensorParserConfig;
+ private String sampleData;
+
+ public SensorParserConfig getSensorParserConfig() {
+ return sensorParserConfig;
+ }
+
+ public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
+ this.sensorParserConfig = sensorParserConfig;
+ }
+
+ public String getSampleData() {
+ return sampleData;
+ }
+
+ public void setSampleData(String sampleData) {
+ this.sampleData = sampleData;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java
new file mode 100644
index 0000000..5f9de62
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public class RestError {
+
+ private int responseCode;
+ private String message;
+ private String fullMessage;
+
+ public RestError(int responseCode, String message, String fullMessage) {
+ this.responseCode = responseCode;
+ this.message = message;
+ this.fullMessage = fullMessage;
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public String getFullMessage() {
+ return fullMessage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java
new file mode 100644
index 0000000..a1b9f03
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public class StellarFunctionDescription {
+
+ private String name;
+ private String description;
+ private String[] params;
+ private String returns;
+
+ public StellarFunctionDescription(String name, String description, String[] params, String returns) {
+ this.name = name;
+ this.description = description;
+ this.params = params;
+ this.returns = returns;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public String[] getParams() {
+ if (params == null) {
+ return new String[0];
+ }
+ return params;
+ }
+
+ public String getReturns() {
+ return returns;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
new file mode 100644
index 0000000..6894e89
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public class TopologyResponse {
+
+ private TopologyResponseCode status;
+ private String message;
+
+ public TopologyResponseCode getStatus() {
+ return status;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setSuccessMessage(String message) {
+ this.status = TopologyResponseCode.SUCCESS;
+ this.message = message;
+ }
+
+ public void setErrorMessage(String message) {
+ this.status = TopologyResponseCode.ERROR;
+ this.message = message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java
new file mode 100644
index 0000000..a85bd89
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public enum TopologyResponseCode {
+
+ SUCCESS,ERROR
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
new file mode 100644
index 0000000..0e2d28a
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import java.util.List;
+import java.util.Map;
+
+public class TopologyStatus {
+
+ private String id;
+ private String name;
+ private TopologyStatusCode status;
+ private Map<String, Object>[] topologyStats;
+ private double latency = 0;
+ private double throughput = 0;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public TopologyStatusCode getStatus() {
+ return status;
+ }
+
+ public void setStatus(TopologyStatusCode status) {
+ this.status = status;
+ }
+
+ public double getLatency() {
+ return latency;
+ }
+
+ public double getThroughput() {
+ return throughput;
+ }
+
+ public void setTopologyStats(List<Map<String, Object>> topologyStats) {
+ for(Map<String, Object> topologyStatsItem: topologyStats) {
+ if ("600".equals(topologyStatsItem.get("window"))) {
+ latency = Double.parseDouble((String) topologyStatsItem.get("completeLatency"));
+ int acked = 0;
+ if (topologyStatsItem.get("acked") != null) {
+ acked = (int) topologyStatsItem.get("acked");
+ }
+ throughput = acked / 600.00;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java
new file mode 100644
index 0000000..461a4ab
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public enum TopologyStatusCode {
+
+ STARTED,
+ STOPPED,
+ ACTIVE,
+ INACTIVE,
+ KILLED,
+ GLOBAL_CONFIG_MISSING,
+ SENSOR_PARSER_CONFIG_MISSING,
+ START_ERROR,
+ STOP_ERROR,
+ TOPOLOGY_NOT_FOUND;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
new file mode 100644
index 0000000..b9d39a4
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+public class TopologySummary {
+
+ private TopologyStatus[] topologies;
+
+ public TopologyStatus[] getTopologies() {
+ if (topologies == null) {
+ return new TopologyStatus[0];
+ }
+ return topologies;
+ }
+
+ public void setTopologies(TopologyStatus[] topologies) {
+ this.topologies = topologies;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
new file mode 100644
index 0000000..c2e39e4
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TransformationValidation {
+
+ private Map<String, Object> sampleData;
+ private SensorParserConfig sensorParserConfig;
+
+ public Map<String, Object> getSampleData() {
+ if (sampleData == null) {
+ return new HashMap<>();
+ }
+ return sampleData;
+ }
+
+ public void setSampleData(Map<String, Object> sampleData) {
+ this.sampleData = sampleData;
+ }
+
+ public SensorParserConfig getSensorParserConfig() {
+ if (sensorParserConfig == null) {
+ return new SensorParserConfig();
+ }
+ return sensorParserConfig;
+ }
+
+ public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
+ this.sensorParserConfig = sensorParserConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
new file mode 100644
index 0000000..63a2d34
--- /dev/null
+++ b/metron-interface/metron-rest/README.md
@@ -0,0 +1,439 @@
+# Metron REST and Configuration UI
+
+This UI exposes and aids in sensor configuration.
+
+## Prerequisites
+
+* A running Metron cluster
+* A running instance of MySQL
+* Java 8 installed
+* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
+
+## Installation
+1. Package the Application with Maven:
+ ```
+ mvn clean package
+ ```
+
+1. Untar the archive in the target directory. The directory structure will look like:
+ ```
+ bin
+ start.sh
+ lib
+ metron-rest-version.jar
+ ```
+
+1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
+ ```
+ export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
+ ```
+
+1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
+ ```
+ export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
+ ```
+
+1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+
+1. Create a Config UI database in MySQL with this command:
+ ```
+ CREATE DATABASE IF NOT EXISTS metronrest
+ ```
+
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+
+1. Start the UI with this command:
+ ```
+ ./bin/start.sh /path/to/application.yml
+ ```
+
+## Usage
+
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement:
+```
+use metronrest;
+insert into users (username, password, enabled) values ('your_username','your_password',1);
+insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
+```
+Users can be added to additional groups with this SQL statement:
+```
+use metronrest;
+insert into authorities (username, authority) values ('your_username', 'your_group');
+```
+
+## API
+
+Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI.
+
+| |
+| ---------- |
+| [ `GET /api/v1/global/config`](#get-apiv1globalconfig)|
+| [ `DELETE /api/v1/global/config`](#delete-apiv1globalconfig)|
+| [ `POST /api/v1/global/config`](#post-apiv1globalconfig)|
+| [ `GET /api/v1/grok/list`](#get-apiv1groklist)|
+| [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)|
+| [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)|
+| [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)|
+| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})|
+| [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopic{name})|
+| [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopic{name}sample)|
+| [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
+| [ `GET /api/v1/sensor/enrichment/config/list/available`](#get-apiv1sensorenrichmentconfiglistavailable)|
+| [ `DELETE /api/v1/sensor/enrichment/config/{name}`](#delete-apiv1sensorenrichmentconfig{name})|
+| [ `POST /api/v1/sensor/enrichment/config/{name}`](#post-apiv1sensorenrichmentconfig{name})|
+| [ `GET /api/v1/sensor/enrichment/config/{name}`](#get-apiv1sensorenrichmentconfig{name})|
+| [ `GET /api/v1/sensor/indexing/config`](#get-apiv1sensorindexingconfig)|
+| [ `DELETE /api/v1/sensor/indexing/config/{name}`](#delete-apiv1sensorindexingconfig{name})|
+| [ `POST /api/v1/sensor/indexing/config/{name}`](#post-apiv1sensorindexingconfig{name})|
+| [ `GET /api/v1/sensor/indexing/config/{name}`](#get-apiv1sensorindexingconfig{name})|
+| [ `POST /api/v1/sensor/parser/config`](#post-apiv1sensorparserconfig)|
+| [ `GET /api/v1/sensor/parser/config`](#get-apiv1sensorparserconfig)|
+| [ `GET /api/v1/sensor/parser/config/list/available`](#get-apiv1sensorparserconfiglistavailable)|
+| [ `POST /api/v1/sensor/parser/config/parseMessage`](#post-apiv1sensorparserconfigparsemessage)|
+| [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)|
+| [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})|
+| [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})|
+| [ `GET /api/v1/storm`](#get-apiv1storm)|
+| [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)|
+| [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)|
+| [ `GET /api/v1/storm/enrichment/activate`](#get-apiv1stormenrichmentactivate)|
+| [ `GET /api/v1/storm/enrichment/deactivate`](#get-apiv1stormenrichmentdeactivate)|
+| [ `GET /api/v1/storm/enrichment/start`](#get-apiv1stormenrichmentstart)|
+| [ `GET /api/v1/storm/enrichment/stop`](#get-apiv1stormenrichmentstop)|
+| [ `GET /api/v1/storm/indexing`](#get-apiv1stormindexing)|
+| [ `GET /api/v1/storm/indexing/activate`](#get-apiv1stormindexingactivate)|
+| [ `GET /api/v1/storm/indexing/deactivate`](#get-apiv1stormindexingdeactivate)|
+| [ `GET /api/v1/storm/indexing/start`](#get-apiv1stormindexingstart)|
+| [ `GET /api/v1/storm/indexing/stop`](#get-apiv1stormindexingstop)|
+| [ `GET /api/v1/storm/parser/activate/{name}`](#get-apiv1stormparseractivate{name})|
+| [ `GET /api/v1/storm/parser/deactivate/{name}`](#get-apiv1stormparserdeactivate{name})|
+| [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})|
+| [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})|
+| [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})|
+| [ `GET /api/v1/transformation/list`](#get-apiv1transformationlist)|
+| [ `GET /api/v1/transformation/list/functions`](#get-apiv1transformationlistfunctions)|
+| [ `GET /api/v1/transformation/list/simple/functions`](#get-apiv1transformationlistsimplefunctions)|
+| [ `POST /api/v1/transformation/validate`](#post-apiv1transformationvalidate)|
+| [ `POST /api/v1/transformation/validate/rules`](#post-apiv1transformationvalidaterules)|
+| [ `GET /api/v1/user`](#get-apiv1user)|
+
+### `GET /api/v1/global/config`
+ * Description: Retrieves the current Global Config from Zookeeper
+ * Returns:
+ * 200 - Returns current Global Config JSON in Zookeeper
+ * 404 - Global Config JSON was not found in Zookeeper
+
+### `DELETE /api/v1/global/config`
+ * Description: Deletes the current Global Config from Zookeeper
+ * Returns:
+ * 200 - Global Config JSON was deleted
+ * 404 - Global Config JSON was not found in Zookeeper
+
+### `POST /api/v1/global/config`
+ * Description: Creates or updates the Global Config in Zookeeper
+ * Input:
+ * globalConfig - The Global Config JSON to be saved
+ * Returns:
+ * 200 - Global Config updated. Returns saved Global Config JSON
+ * 201 - Global Config created. Returns saved Global Config JSON
+
+### `GET /api/v1/grok/list`
+ * Description: Lists the common Grok statements available in Metron
+ * Returns:
+ * 200 - JSON object containing pattern label/Grok statements key value pairs
+
+### `POST /api/v1/grok/validate`
+ * Description: Applies a Grok statement to a sample message
+ * Input:
+ * grokValidation - Object containing Grok statment and sample message
+ * Returns:
+ * 200 - JSON results
+
+### `GET /api/v1/kafka/topic`
+ * Description: Retrieves all Kafka topics
+ * Returns:
+ * 200 - Returns a list of all Kafka topics
+
+### `POST /api/v1/kafka/topic`
+ * Description: Creates a new Kafka topic
+ * Input:
+ * topic - Kafka topic
+ * Returns:
+ * 200 - Returns saved Kafka topic
+
+### `GET /api/v1/kafka/topic/{name}`
+ * Description: Retrieves a Kafka topic
+ * Input:
+ * name - Kafka topic name
+ * Returns:
+ * 200 - Returns Kafka topic
+ * 404 - Kafka topic is missing
+
+### `DELETE /api/v1/kafka/topic/{name}`
+ * Description: Delets a Kafka topic
+ * Input:
+ * name - Kafka topic name
+ * Returns:
+ * 200 - Kafka topic was deleted
+ * 404 - Kafka topic is missing
+
+### `GET /api/v1/kafka/topic/{name}/sample`
+ * Description: Retrieves a sample message from a Kafka topic using the most recent offset
+ * Input:
+ * name - Kafka topic name
+ * Returns:
+ * 200 - Returns sample message
+ * 404 - Either Kafka topic is missing or contains no messages
+
+### `GET /api/v1/sensor/enrichment/config`
+ * Description: Retrieves all SensorEnrichmentConfigs from Zookeeper
+ * Returns:
+ * 200 - Returns all SensorEnrichmentConfigs
+
+### `GET /api/v1/sensor/enrichment/config/list/available`
+ * Description: Lists the available enrichments
+ * Returns:
+ * 200 - Returns a list of available enrichments
+
+### `DELETE /api/v1/sensor/enrichment/config/{name}`
+ * Description: Deletes a SensorEnrichmentConfig from Zookeeper
+ * Input:
+ * name - SensorEnrichmentConfig name
+ * Returns:
+ * 200 - SensorEnrichmentConfig was deleted
+ * 404 - SensorEnrichmentConfig is missing
+
+### `POST /api/v1/sensor/enrichment/config/{name}`
+ * Description: Updates or creates a SensorEnrichmentConfig in Zookeeper
+ * Input:
+ * sensorEnrichmentConfig - SensorEnrichmentConfig
+ * name - SensorEnrichmentConfig name
+ * Returns:
+ * 200 - SensorEnrichmentConfig updated. Returns saved SensorEnrichmentConfig
+ * 201 - SensorEnrichmentConfig created. Returns saved SensorEnrichmentConfig
+
+### `GET /api/v1/sensor/enrichment/config/{name}`
+ * Description: Retrieves a SensorEnrichmentConfig from Zookeeper
+ * Input:
+ * name - SensorEnrichmentConfig name
+ * Returns:
+ * 200 - Returns SensorEnrichmentConfig
+ * 404 - SensorEnrichmentConfig is missing
+
+### `GET /api/v1/sensor/indexing/config`
+ * Description: Retrieves all SensorIndexingConfigs from Zookeeper
+ * Returns:
+ * 200 - Returns all SensorIndexingConfigs
+
+### `DELETE /api/v1/sensor/indexing/config/{name}`
+ * Description: Deletes a SensorIndexingConfig from Zookeeper
+ * Input:
+ * name - SensorIndexingConfig name
+ * Returns:
+ * 200 - SensorIndexingConfig was deleted
+ * 404 - SensorIndexingConfig is missing
+
+### `POST /api/v1/sensor/indexing/config/{name}`
+ * Description: Updates or creates a SensorIndexingConfig in Zookeeper
+ * Input:
+ * sensorIndexingConfig - SensorIndexingConfig
+ * name - SensorIndexingConfig name
+ * Returns:
+ * 200 - SensorIndexingConfig updated. Returns saved SensorIndexingConfig
+ * 201 - SensorIndexingConfig created. Returns saved SensorIndexingConfig
+
+### `GET /api/v1/sensor/indexing/config/{name}`
+ * Description: Retrieves a SensorIndexingConfig from Zookeeper
+ * Input:
+ * name - SensorIndexingConfig name
+ * Returns:
+ * 200 - Returns SensorIndexingConfig
+ * 404 - SensorIndexingConfig is missing
+
+### `POST /api/v1/sensor/parser/config`
+ * Description: Updates or creates a SensorParserConfig in Zookeeper
+ * Input:
+ * sensorParserConfig - SensorParserConfig
+ * Returns:
+ * 200 - SensorParserConfig updated. Returns saved SensorParserConfig
+ * 201 - SensorParserConfig created. Returns saved SensorParserConfig
+
+### `GET /api/v1/sensor/parser/config`
+ * Description: Retrieves all SensorParserConfigs from Zookeeper
+ * Returns:
+ * 200 - Returns all SensorParserConfigs
+
+### `GET /api/v1/sensor/parser/config/list/available`
+ * Description: Lists the available parser classes that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of available parser classes
+
+### `POST /api/v1/sensor/parser/config/parseMessage`
+ * Description: Parses a sample message given a SensorParserConfig
+ * Input:
+ * parseMessageRequest - Object containing a sample message and SensorParserConfig
+ * Returns:
+ * 200 - Returns parsed message
+
+### `GET /api/v1/sensor/parser/config/reload/available`
+ * Description: Scans the classpath for available parser classes and reloads the cached parser class list
+ * Returns:
+ * 200 - Returns a list of available parser classes
+
+### `DELETE /api/v1/sensor/parser/config/{name}`
+ * Description: Deletes a SensorParserConfig from Zookeeper
+ * Input:
+ * name - SensorParserConfig name
+ * Returns:
+ * 200 - SensorParserConfig was deleted
+ * 404 - SensorParserConfig is missing
+
+### `GET /api/v1/sensor/parser/config/{name}`
+ * Description: Retrieves a SensorParserConfig from Zookeeper
+ * Input:
+ * name - SensorParserConfig name
+ * Returns:
+ * 200 - Returns SensorParserConfig
+ * 404 - SensorParserConfig is missing
+
+### `GET /api/v1/storm`
+ * Description: Retrieves the status of all Storm topologies
+ * Returns:
+ * 200 - Returns a list of topologies with status information
+
+### `GET /api/v1/storm/client/status`
+ * Description: Retrieves information about the Storm command line client
+ * Returns:
+ * 200 - Returns storm command line client information
+
+### `GET /api/v1/storm/enrichment`
+ * Description: Retrieves the status of the Storm enrichment topology
+ * Returns:
+ * 200 - Returns topology status information
+ * 404 - Topology is missing
+
+### `GET /api/v1/storm/enrichment/activate`
+ * Description: Activates a Storm enrichment topology
+ * Returns:
+ * 200 - Returns activate response message
+
+### `GET /api/v1/storm/enrichment/deactivate`
+ * Description: Deactivates a Storm enrichment topology
+ * Returns:
+ * 200 - Returns deactivate response message
+
+### `GET /api/v1/storm/enrichment/start`
+ * Description: Starts a Storm enrichment topology
+ * Returns:
+ * 200 - Returns start response message
+
+### `GET /api/v1/storm/enrichment/stop`
+ * Description: Stops a Storm enrichment topology
+ * Input:
+ * stopNow - Stop the topology immediately
+ * Returns:
+ * 200 - Returns stop response message
+
+### `GET /api/v1/storm/indexing`
+ * Description: Retrieves the status of the Storm indexing topology
+ * Returns:
+ * 200 - Returns topology status information
+ * 404 - Topology is missing
+
+### `GET /api/v1/storm/indexing/activate`
+ * Description: Activates a Storm indexing topology
+ * Returns:
+ * 200 - Returns activate response message
+
+### `GET /api/v1/storm/indexing/deactivate`
+ * Description: Deactivates a Storm indexing topology
+ * Returns:
+ * 200 - Returns deactivate response message
+
+### `GET /api/v1/storm/indexing/start`
+ * Description: Starts a Storm indexing topology
+ * Returns:
+ * 200 - Returns start response message
+
+### `GET /api/v1/storm/indexing/stop`
+ * Description: Stops a Storm enrichment topology
+ * Input:
+ * stopNow - Stop the topology immediately
+ * Returns:
+ * 200 - Returns stop response message
+
+### `GET /api/v1/storm/parser/activate/{name}`
+ * Description: Activates a Storm parser topology
+ * Input:
+ * name - Parser name
+ * Returns:
+ * 200 - Returns activate response message
+
+### `GET /api/v1/storm/parser/deactivate/{name}`
+ * Description: Deactivates a Storm parser topology
+ * Input:
+ * name - Parser name
+ * Returns:
+ * 200 - Returns deactivate response message
+
+### `GET /api/v1/storm/parser/start/{name}`
+ * Description: Starts a Storm parser topology
+ * Input:
+ * name - Parser name
+ * Returns:
+ * 200 - Returns start response message
+
+### `GET /api/v1/storm/parser/stop/{name}`
+ * Description: Stops a Storm parser topology
+ * Input:
+ * name - Parser name
+ * stopNow - Stop the topology immediately
+ * Returns:
+ * 200 - Returns stop response message
+
+### `GET /api/v1/storm/{name}`
+ * Description: Retrieves the status of a Storm topology
+ * Input:
+ * name - Topology name
+ * Returns:
+ * 200 - Returns topology status information
+ * 404 - Topology is missing
+
+### `GET /api/v1/transformation/list`
+ * Description: Retrieves field transformations
+ * Returns:
+ * 200 - Returns a list field transformations
+
+### `GET /api/v1/transformation/list/functions`
+ * Description: Lists the Stellar functions that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of Stellar functions
+
+### `GET /api/v1/transformation/list/simple/functions`
+ * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of simple Stellar functions
+
+### `POST /api/v1/transformation/validate`
+ * Description: Executes transformations against a sample message
+ * Input:
+ * transformationValidation - Object containing SensorParserConfig and sample message
+ * Returns:
+ * 200 - Returns transformation results
+
+### `POST /api/v1/transformation/validate/rules`
+ * Description: Tests Stellar statements to ensure they are well-formed
+ * Input:
+ * statements - List of statements to validate
+ * Returns:
+ * 200 - Returns validation results
+
+### `GET /api/v1/user`
+ * Description: Retrieves the current user
+ * Returns:
+ * 200 - Current user
+
+
+## License
+
+This project depends on the Java Transaction API. See https://java.net/projects/jta-spec/ for more details.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
new file mode 100644
index 0000000..c2beb2d
--- /dev/null
+++ b/metron-interface/metron-rest/pom.xml
@@ -0,0 +1,328 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-interface</artifactId>
+ <version>0.3.0</version>
+ </parent>
+ <artifactId>metron-rest</artifactId>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <java.version>1.8</java.version>
+ <antlr.version>4.5</antlr.version>
+ <curator.version>2.7.1</curator.version>
+ <powermock.version>1.6.4</powermock.version>
+ <spring.boot.version>1.4.1.RELEASE</spring.boot.version>
+ <swagger.version>2.5.0</swagger.version>
+ <mysql.client.version>5.1.40</mysql.client.version>
+ <emma.version>1.0-alpha-3</emma.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.hibernate</groupId>
+ <artifactId>hibernate-validator</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-jdbc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.client.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>${global_json_simple_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-runtime</artifactId>
+ <version>${antlr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.8.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-joda</artifactId>
+ <version>2.8.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-rest-client</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-swagger2</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-swagger-ui</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${global_kafka_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.thekraken</groupId>
+ <artifactId>grok</artifactId>
+ <version>0.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ <version>1.7</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <!-- Import dependency management from Spring Boot -->
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>${emma.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <targetJdk>${global_java_version}</targetJdk>
+ </configuration>
+ </plugin>
+ </plugins>
+ </reporting>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring.boot.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <layout>ZIP</layout>
+ <excludes>
+ <exclude>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </exclude>
+ <exclude>
+ <groupId>org.hibernate.common</groupId>
+ <artifactId>hibernate-commons-annotations</artifactId>
+ </exclude>
+ <exclude>
+ <groupId>org.hibernate</groupId>
+ <artifactId>hibernate-core</artifactId>
+ </exclude>
+ <exclude>
+ <groupId>org.hibernate</groupId>
+ <artifactId>hibernate-envers</artifactId>
+ </exclude>
+ <exclude>
+ <groupId>org.hibernate</groupId>
+ <artifactId>hibernate-entitymanager</artifactId>
+ </exclude>
+ <exclude>
+ <groupId>org.hibernate.javax.persistence</groupId>
+ <artifactId>hibernate-jpa-2.1-api</artifactId>
+ </exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/assembly/assembly.xml b/metron-interface/metron-rest/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..7a5dbee
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/assembly/assembly.xml
@@ -0,0 +1,54 @@
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>/config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java
new file mode 100644
index 0000000..ac89b1d
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class MetronRestApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(MetronRestApplication.class, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
new file mode 100644
index 0000000..8786222
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest;
+
+import org.apache.metron.parsers.GrokParser;
+
+public class MetronRestConstants {
+
+ public static final String DEV_PROFILE = "dev";
+ public static final String TEST_PROFILE = "test";
+ public static final String DOCKER_PROFILE = "docker";
+ public static final String CSRF_ENABLE_PROFILE = "csrf-enable";
+
+ public static final String GROK_DEFAULT_PATH_SPRING_PROPERTY = "grok.path.default";
+ public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp";
+ public static final String GROK_CLASS_NAME = GrokParser.class.getName();
+ public static final String GROK_PATH_KEY = "grokPath";
+ public static final String GROK_STATEMENT_KEY = "grokStatement";
+ public static final String GROK_PATTERN_LABEL_KEY = "patternLabel";
+
+ public static final String STORM_UI_SPRING_PROPERTY = "storm.ui.url";
+ public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary";
+ public static final String TOPOLOGY_URL = "/api/v1/topology";
+ public static final String ENRICHMENT_TOPOLOGY_NAME = "enrichment";
+ public static final String INDEXING_TOPOLOGY_NAME = "indexing";
+ public static final String PARSER_SCRIPT_PATH_SPRING_PROPERTY = "storm.parser.script.path";
+ public static final String ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY = "storm.enrichment.script.path";
+ public static final String INDEXING_SCRIPT_PATH_SPRING_PROPERTY = "storm.indexing.script.path";
+
+ public static final String ZK_URL_SPRING_PROPERTY = "zookeeper.url";
+ public static final String ZK_CLIENT_SESSION_TIMEOUT = "zookeeper.client.timeout.session";
+ public static final String ZK_CLIENT_CONNECTION_TIMEOUT = "zookeeper.client.timeout.connection";
+ public static final String CURATOR_SLEEP_TIME = "curator.sleep.time";
+ public static final String CURATOR_MAX_RETRIES = "curator.max.retries";
+
+ public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url";
+
+ public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url";
+ public static final String DEFAULT_HDFS_URL = "file:///";
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java
new file mode 100644
index 0000000..1f985f7
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest;
+
+public class RestException extends Exception {
+
+ private String fullMessage;
+
+ public RestException(Exception e) { super( e.getMessage(), e.getCause()); }
+
+ public RestException(String message) {
+ super(message);
+ }
+
+ public RestException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public RestException(String message, String fullMessage, Throwable cause) {
+ super(message, cause);
+ this.fullMessage = fullMessage;
+ }
+
+ public String getFullMessage() {
+ return this.fullMessage;
+ }
+
+}
[04/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
new file mode 100644
index 0000000..efda639
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.ParseMessageRequest;
+import org.json.simple.JSONObject;
+
+import java.util.Map;
+
+public interface SensorParserConfigService {
+
+ SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException;
+
+ SensorParserConfig findOne(String name) throws RestException;
+
+ Iterable<SensorParserConfig> getAll() throws RestException;
+
+ boolean delete(String name) throws RestException;
+
+ Map<String, String> getAvailableParsers();
+
+ Map<String, String> reloadAvailableParsers();
+
+ JSONObject parseMessage(ParseMessageRequest parseMessageRequest) throws RestException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
new file mode 100644
index 0000000..8c1e228
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.TopologyResponse;
+
+import java.util.Map;
+
+public interface StormAdminService {
+
+ TopologyResponse startParserTopology(String name) throws RestException;
+
+ TopologyResponse stopParserTopology(String name, boolean stopNow) throws RestException;
+
+ TopologyResponse startEnrichmentTopology() throws RestException;
+
+ TopologyResponse stopEnrichmentTopology(boolean stopNow) throws RestException;
+
+ TopologyResponse startIndexingTopology() throws RestException;
+
+ TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException;
+
+ Map<String, String> getStormClientStatus() throws RestException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java
new file mode 100644
index 0000000..76216d0
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatus;
+import org.apache.metron.rest.model.TopologySummary;
+
+import java.util.List;
+
+public interface StormStatusService {
+
+ TopologySummary getTopologySummary();
+
+ TopologyStatus getTopologyStatus(String name);
+
+ List<TopologyStatus> getAllTopologyStatus();
+
+ TopologyResponse activateTopology(String name);
+
+ TopologyResponse deactivateTopology(String name);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
new file mode 100644
index 0000000..d1400c6
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.TransformationValidation;
+
+import java.util.List;
+import java.util.Map;
+
+public interface TransformationService {
+
+ Map<String, Boolean> validateRules(List<String> rules);
+
+ Map<String, Object> validateTransformation(TransformationValidation transformationValidation);
+
+ FieldTransformations[] getTransformations();
+
+ List<StellarFunctionDescription> getStellarFunctions();
+
+ List<StellarFunctionDescription> getSimpleStellarFunctions();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
new file mode 100644
index 0000000..d7bbb23
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+public class DockerStormCLIWrapper extends StormCLIWrapper {
+
+ private Logger LOG = LoggerFactory.getLogger(DockerStormCLIWrapper.class);
+
+ @Autowired
+ private Environment environment;
+
+ @Override
+ protected ProcessBuilder getProcessBuilder(String... command) {
+ String[] dockerCommand = {"docker-compose", "-f", environment.getProperty("docker.compose.path"), "-p", "metron", "exec", "storm"};
+ ProcessBuilder pb = new ProcessBuilder(ArrayUtils.addAll(dockerCommand, command));
+ Map<String, String> pbEnvironment = pb.environment();
+ pbEnvironment.put("METRON_VERSION", environment.getProperty("metron.version"));
+ setDockerEnvironment(pbEnvironment);
+ return pb;
+ }
+
+ protected void setDockerEnvironment(Map<String, String> environmentVariables) {
+ ProcessBuilder pb = getDockerEnvironmentProcessBuilder();
+ try {
+ Process process = pb.start();
+ BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream()));
+ String line;
+ while((line = inputStream.readLine()) != null) {
+ if (line.startsWith("export")) {
+ String[] parts = line.replaceFirst("export ", "").split("=");
+ environmentVariables.put(parts[0], parts[1].replaceAll("\"", ""));
+ }
+ }
+ process.waitFor();
+ } catch (IOException | InterruptedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ protected ProcessBuilder getDockerEnvironmentProcessBuilder() {
+ String[] command = {"docker-machine", "env", "metron-machine"};
+ return new ProcessBuilder(command);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
new file mode 100644
index 0000000..54c331a
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+
+@Service
+public class GlobalConfigServiceImpl implements GlobalConfigService {
+
+ @Autowired
+ private CuratorFramework client;
+
+ @Override
+ public Map<String, Object> save(Map<String, Object> globalConfig) throws RestException {
+ try {
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, client);
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return globalConfig;
+ }
+
+ @Override
+ public Map<String, Object> get() throws RestException {
+ Map<String, Object> globalConfig;
+ try {
+ byte[] globalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client);
+ globalConfig = JSONUtils.INSTANCE.load(new ByteArrayInputStream(globalConfigBytes), new TypeReference<Map<String, Object>>(){});
+ } catch (KeeperException.NoNodeException e) {
+ return null;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return globalConfig;
+ }
+
+ @Override
+ public boolean delete() throws RestException {
+ try {
+ client.delete().forPath(ConfigurationType.GLOBAL.getZookeeperRoot());
+ } catch (KeeperException.NoNodeException e) {
+ return false;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
new file mode 100644
index 0000000..8fbea13
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.GrokValidation;
+import org.apache.metron.rest.service.GrokService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.util.Map;
+
+@Service
+public class GrokServiceImpl implements GrokService {
+
+ @Autowired
+ private Grok commonGrok;
+
+ @Override
+ public Map<String, String> getCommonGrokPatterns() {
+ return commonGrok.getPatterns();
+ }
+
+ @Override
+ public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException {
+ Map<String, Object> results;
+ try {
+ Grok grok = new Grok();
+ grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
+ grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
+ String patternLabel = grokValidation.getStatement().substring(0, grokValidation.getStatement().indexOf(" "));
+ String grokPattern = "%{" + patternLabel + "}";
+ grok.compile(grokPattern);
+ Match gm = grok.match(grokValidation.getSampleData());
+ gm.captures();
+ results = gm.toMap();
+ results.remove(patternLabel);
+ } catch (StringIndexOutOfBoundsException e) {
+ throw new RestException("A pattern label must be included (ex. PATTERN_LABEL ${PATTERN:field} ...)", e.getCause());
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ grokValidation.setResults(results);
+ return grokValidation;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
new file mode 100644
index 0000000..c14ec0c
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.metron.rest.service.HdfsService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+@Service
+public class HdfsServiceImpl implements HdfsService {
+
+ @Autowired
+ private Configuration configuration;
+
+ @Override
+ public byte[] read(Path path) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
+ return byteArrayOutputStream.toByteArray();
+ }
+
+ @Override
+ public void write(Path path, byte[] contents) throws IOException {
+ FSDataOutputStream fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+ fsDataOutputStream.write(contents);
+ fsDataOutputStream.close();
+ }
+
+ @Override
+ public FileStatus[] list(Path path) throws IOException {
+ return FileSystem.get(configuration).listStatus(path);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ return FileSystem.get(configuration).delete(path, recursive);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
new file mode 100644
index 0000000..7246c2f
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import kafka.admin.AdminOperationException;
+import kafka.admin.AdminUtils;
+import kafka.admin.RackAwareMode;
+import kafka.utils.ZkUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.KafkaTopic;
+import org.apache.metron.rest.service.KafkaService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Service
+public class KafkaServiceImpl implements KafkaService {
+
+ @Autowired
+ private ZkUtils zkUtils;
+
+ @Autowired
+ private KafkaConsumer<String, String> kafkaConsumer;
+
+ private String offsetTopic = "__consumer_offsets";
+
+ @Override
+ public KafkaTopic createTopic(KafkaTopic topic) throws RestException {
+ if (!listTopics().contains(topic.getName())) {
+ try {
+ AdminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$);
+ } catch (AdminOperationException e) {
+ throw new RestException(e);
+ }
+ }
+ return topic;
+ }
+
+ @Override
+ public boolean deleteTopic(String name) {
+ if (listTopics().contains(name)) {
+ AdminUtils.deleteTopic(zkUtils, name);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public KafkaTopic getTopic(String name) {
+ KafkaTopic kafkaTopic = null;
+ if (listTopics().contains(name)) {
+ List<PartitionInfo> partitionInfos = kafkaConsumer.partitionsFor(name);
+ if (partitionInfos.size() > 0) {
+ PartitionInfo partitionInfo = partitionInfos.get(0);
+ kafkaTopic = new KafkaTopic();
+ kafkaTopic.setName(name);
+ kafkaTopic.setNumPartitions(partitionInfos.size());
+ kafkaTopic.setReplicationFactor(partitionInfo.replicas().length);
+ }
+ }
+ return kafkaTopic;
+ }
+
+ @Override
+ public Set<String> listTopics() {
+ Set<String> topics;
+ synchronized (this) {
+ topics = kafkaConsumer.listTopics().keySet();
+ topics.remove(offsetTopic);
+ }
+ return topics;
+ }
+
+ @Override
+ public String getSampleMessage(String topic) {
+ String message = null;
+ if (listTopics().contains(topic)) {
+ synchronized (this) {
+ kafkaConsumer.assign(kafkaConsumer.partitionsFor(topic).stream().map(partitionInfo ->
+ new TopicPartition(topic, partitionInfo.partition())).collect(Collectors.toList()));
+ for (TopicPartition topicPartition : kafkaConsumer.assignment()) {
+ long offset = kafkaConsumer.position(topicPartition) - 1;
+ if (offset >= 0) {
+ kafkaConsumer.seek(topicPartition, offset);
+ }
+ }
+ ConsumerRecords<String, String> records = kafkaConsumer.poll(100);
+ Iterator<ConsumerRecord<String, String>> iterator = records.iterator();
+ if (iterator.hasNext()) {
+ ConsumerRecord<String, String> record = iterator.next();
+ message = record.value();
+ }
+ kafkaConsumer.unsubscribe();
+ }
+ }
+ return message;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
new file mode 100644
index 0000000..8b3dbb7
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorEnrichmentConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
+
+ @Autowired
+ private ObjectMapper objectMapper;
+
+ @Autowired
+ private CuratorFramework client;
+
+ @Override
+ public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
+ try {
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(name, objectMapper.writeValueAsString(sensorEnrichmentConfig).getBytes(), client);
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorEnrichmentConfig;
+ }
+
+ @Override
+ public SensorEnrichmentConfig findOne(String name) throws RestException {
+ SensorEnrichmentConfig sensorEnrichmentConfig;
+ try {
+ sensorEnrichmentConfig = ConfigurationsUtils.readSensorEnrichmentConfigFromZookeeper(name, client);
+ } catch (KeeperException.NoNodeException e) {
+ return null;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorEnrichmentConfig;
+ }
+
+ @Override
+ public Map<String, SensorEnrichmentConfig> getAll() throws RestException {
+ Map<String, SensorEnrichmentConfig> sensorEnrichmentConfigs = new HashMap<>();
+ List<String> sensorNames = getAllTypes();
+ for (String name : sensorNames) {
+ sensorEnrichmentConfigs.put(name, findOne(name));
+ }
+ return sensorEnrichmentConfigs;
+ }
+
+ @Override
+ public List<String> getAllTypes() throws RestException {
+ List<String> types;
+ try {
+ types = client.getChildren().forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot());
+ } catch (KeeperException.NoNodeException e) {
+ types = new ArrayList<>();
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return types;
+ }
+
+ @Override
+ public boolean delete(String name) throws RestException {
+ try {
+ client.delete().forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/" + name);
+ } catch (KeeperException.NoNodeException e) {
+ return false;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return true;
+ }
+
+ @Override
+ public List<String> getAvailableEnrichments() {
+ return new ArrayList<String>() {{
+ add("geo");
+ add("host");
+ add("whois");
+ }};
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
new file mode 100644
index 0000000..ab46418
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorIndexingConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
+
+ @Autowired
+ private ObjectMapper objectMapper;
+
+ @Autowired
+ private CuratorFramework client;
+
+ @Override
+ public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException {
+ try {
+ ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(name, objectMapper.writeValueAsString(sensorIndexingConfig).getBytes(), client);
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorIndexingConfig;
+ }
+
+ @Override
+ public Map<String, Object> findOne(String name) throws RestException {
+ Map<String, Object> sensorIndexingConfig;
+ try {
+ byte[] sensorIndexingConfigBytes = ConfigurationsUtils.readSensorIndexingConfigBytesFromZookeeper(name, client);
+ sensorIndexingConfig = JSONUtils.INSTANCE.load(new ByteArrayInputStream(sensorIndexingConfigBytes), new TypeReference<Map<String, Object>>(){});
+ } catch (KeeperException.NoNodeException e) {
+ return null;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorIndexingConfig;
+ }
+
+ @Override
+ public Map<String, Map<String, Object>> getAll() throws RestException {
+ Map<String, Map<String, Object>> sensorIndexingConfigs = new HashMap<>();
+ List<String> sensorNames = getAllTypes();
+ for (String name : sensorNames) {
+ sensorIndexingConfigs.put(name, findOne(name));
+ }
+ return sensorIndexingConfigs;
+ }
+
+ @Override
+ public List<String> getAllTypes() throws RestException {
+ List<String> types;
+ try {
+ types = client.getChildren().forPath(ConfigurationType.INDEXING.getZookeeperRoot());
+ } catch (KeeperException.NoNodeException e) {
+ types = new ArrayList<>();
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return types;
+ }
+
+ @Override
+ public boolean delete(String name) throws RestException {
+ try {
+ client.delete().forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/" + name);
+ } catch (KeeperException.NoNodeException e) {
+ return false;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
new file mode 100644
index 0000000..cb88708
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.ParseMessageRequest;
+import org.apache.metron.rest.service.HdfsService;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.json.simple.JSONObject;
+import org.reflections.Reflections;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
+import static org.apache.metron.rest.MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY;
+import static org.apache.metron.rest.MetronRestConstants.GROK_PATH_KEY;
+import static org.apache.metron.rest.MetronRestConstants.GROK_PATTERN_LABEL_KEY;
+import static org.apache.metron.rest.MetronRestConstants.GROK_STATEMENT_KEY;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+
+@Service
+public class SensorParserConfigServiceImpl implements SensorParserConfigService {
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private ObjectMapper objectMapper;
+
+ private CuratorFramework client;
+
+ @Autowired
+ public void setClient(CuratorFramework client) {
+ this.client = client;
+ }
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ private Map<String, String> availableParsers;
+
+ @Override
+ public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
+ String serializedConfig;
+ if (isGrokConfig(sensorParserConfig)) {
+ addGrokPathToConfig(sensorParserConfig);
+ sensorParserConfig.getParserConfig().putIfAbsent(MetronRestConstants.GROK_PATTERN_LABEL_KEY, sensorParserConfig.getSensorTopic().toUpperCase());
+ String statement = (String) sensorParserConfig.getParserConfig().remove(MetronRestConstants.GROK_STATEMENT_KEY);
+ serializedConfig = serialize(sensorParserConfig);
+ sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_STATEMENT_KEY, statement);
+ saveGrokStatement(sensorParserConfig);
+ } else {
+ serializedConfig = serialize(sensorParserConfig);
+ }
+ try {
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), serializedConfig.getBytes(), client);
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorParserConfig;
+ }
+
+ private String serialize(SensorParserConfig sensorParserConfig) throws RestException {
+ String serializedConfig;
+ try {
+ serializedConfig = objectMapper.writeValueAsString(sensorParserConfig);
+ } catch (JsonProcessingException e) {
+ throw new RestException("Could not serialize SensorParserConfig", "Could not serialize " + sensorParserConfig.toString(), e.getCause());
+ }
+ return serializedConfig;
+ }
+
+ @Override
+ public SensorParserConfig findOne(String name) throws RestException {
+ SensorParserConfig sensorParserConfig;
+ try {
+ sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client);
+ if (isGrokConfig(sensorParserConfig)) {
+ addGrokStatementToConfig(sensorParserConfig);
+ }
+ } catch (KeeperException.NoNodeException e) {
+ return null;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return sensorParserConfig;
+ }
+
+ @Override
+ public Iterable<SensorParserConfig> getAll() throws RestException {
+ List<SensorParserConfig> sensorParserConfigs = new ArrayList<>();
+ List<String> sensorNames = getAllTypes();
+ for (String name : sensorNames) {
+ sensorParserConfigs.add(findOne(name));
+ }
+ return sensorParserConfigs;
+ }
+
+ @Override
+ public boolean delete(String name) throws RestException {
+ try {
+ client.delete().forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/" + name);
+ } catch (KeeperException.NoNodeException e) {
+ return false;
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return true;
+ }
+
+ private List<String> getAllTypes() throws RestException {
+ List<String> types;
+ try {
+ types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot());
+ } catch (KeeperException.NoNodeException e) {
+ types = new ArrayList<>();
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return types;
+ }
+
+ @Override
+ public Map<String, String> getAvailableParsers() {
+ if (availableParsers == null) {
+ availableParsers = new HashMap<>();
+ Set<Class<? extends MessageParser>> parserClasses = getParserClasses();
+ parserClasses.forEach(parserClass -> {
+ if (!"BasicParser".equals(parserClass.getSimpleName())) {
+ availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), parserClass.getName());
+ }
+ });
+ }
+ return availableParsers;
+ }
+
+ @Override
+ public Map<String, String> reloadAvailableParsers() {
+ availableParsers = null;
+ return getAvailableParsers();
+ }
+
+ private Set<Class<? extends MessageParser>> getParserClasses() {
+ Reflections reflections = new Reflections("org.apache.metron.parsers");
+ return reflections.getSubTypesOf(MessageParser.class);
+ }
+
+ @Override
+ public JSONObject parseMessage(ParseMessageRequest parseMessageRequest) throws RestException {
+ SensorParserConfig sensorParserConfig = parseMessageRequest.getSensorParserConfig();
+ if (sensorParserConfig == null) {
+ throw new RestException("SensorParserConfig is missing from ParseMessageRequest");
+ } else if (sensorParserConfig.getParserClassName() == null) {
+ throw new RestException("SensorParserConfig must have a parserClassName");
+ } else {
+ MessageParser<JSONObject> parser = null;
+ try {
+ parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
+ } catch (Exception e) {
+ throw new RestException(e.toString(), e.getCause());
+ }
+ if (isGrokConfig(sensorParserConfig)) {
+ saveTemporaryGrokStatement(sensorParserConfig);
+ sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()).toString());
+ }
+ parser.configure(sensorParserConfig.getParserConfig());
+ JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
+ if (isGrokConfig(sensorParserConfig)) {
+ deleteTemporaryGrokStatement(sensorParserConfig);
+ }
+ return results;
+ }
+ }
+
+ private boolean isGrokConfig(SensorParserConfig sensorParserConfig) {
+ return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName());
+ }
+
+ private void addGrokStatementToConfig(SensorParserConfig sensorParserConfig) throws RestException {
+ String grokStatement = "";
+ String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
+ if (grokPath != null) {
+ String fullGrokStatement = getGrokStatement(grokPath);
+ String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
+ grokStatement = fullGrokStatement.replaceFirst(patternLabel + " ", "");
+ }
+ sensorParserConfig.getParserConfig().put(GROK_STATEMENT_KEY, grokStatement);
+ }
+
+ private void addGrokPathToConfig(SensorParserConfig sensorParserConfig) {
+ if (sensorParserConfig.getParserConfig().get(GROK_PATH_KEY) == null) {
+ String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
+ if (grokStatement != null) {
+ sensorParserConfig.getParserConfig().put(GROK_PATH_KEY,
+ new Path(environment.getProperty(GROK_DEFAULT_PATH_SPRING_PROPERTY), sensorParserConfig.getSensorTopic()).toString());
+ }
+ }
+ }
+
+ private String getGrokStatement(String path) throws RestException {
+ try {
+ return new String(hdfsService.read(new Path(path)));
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ }
+
+ private void saveGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
+ saveGrokStatement(sensorParserConfig, false);
+ }
+
+ private void saveTemporaryGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
+ saveGrokStatement(sensorParserConfig, true);
+ }
+
+ private void saveGrokStatement(SensorParserConfig sensorParserConfig, boolean isTemporary) throws RestException {
+ String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
+ String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
+ String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
+ if (grokStatement != null) {
+ String fullGrokStatement = patternLabel + " " + grokStatement;
+ try {
+ if (!isTemporary) {
+ hdfsService.write(new Path(grokPath), fullGrokStatement.getBytes());
+ } else {
+ File grokDirectory = new File(getTemporaryGrokRootPath());
+ if (!grokDirectory.exists()) {
+ grokDirectory.mkdirs();
+ }
+ FileWriter fileWriter = new FileWriter(new File(grokDirectory, sensorParserConfig.getSensorTopic()));
+ fileWriter.write(fullGrokStatement);
+ fileWriter.close();
+ }
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ } else {
+ throw new RestException("A grokStatement must be provided");
+ }
+ }
+
+ private void deleteTemporaryGrokStatement(SensorParserConfig sensorParserConfig) {
+ File file = new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic());
+ file.delete();
+ }
+
+ private String getTemporaryGrokRootPath() {
+ String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
+ Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+ return new Path(grokTempPath, authentication.getName()).toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
new file mode 100644
index 0000000..cb7c449
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.apache.metron.rest.service.StormAdminService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+
+@Service
+public class StormAdminServiceImpl implements StormAdminService {
+
+ private StormCLIWrapper stormCLIClientWrapper;
+
+ @Autowired
+ public void setStormCLIClientWrapper(StormCLIWrapper stormCLIClientWrapper) {
+ this.stormCLIClientWrapper = stormCLIClientWrapper;
+ }
+
+ @Autowired
+ private GlobalConfigService globalConfigService;
+
+ @Autowired
+ private SensorParserConfigService sensorParserConfigService;
+
+ @Override
+ public TopologyResponse startParserTopology(String name) throws RestException {
+ TopologyResponse topologyResponse = new TopologyResponse();
+ if (globalConfigService.get() == null) {
+ topologyResponse.setErrorMessage(TopologyStatusCode.GLOBAL_CONFIG_MISSING.toString());
+ } else if (sensorParserConfigService.findOne(name) == null) {
+ topologyResponse.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString());
+ } else {
+ topologyResponse = createResponse(stormCLIClientWrapper.startParserTopology(name), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
+ }
+ return topologyResponse;
+ }
+
+ @Override
+ public TopologyResponse stopParserTopology(String name, boolean stopNow) throws RestException {
+ return createResponse(stormCLIClientWrapper.stopParserTopology(name, stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
+ }
+
+ @Override
+ public TopologyResponse startEnrichmentTopology() throws RestException {
+ return createResponse(stormCLIClientWrapper.startEnrichmentTopology(), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
+ }
+
+ @Override
+ public TopologyResponse stopEnrichmentTopology(boolean stopNow) throws RestException {
+ return createResponse(stormCLIClientWrapper.stopEnrichmentTopology(stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
+ }
+
+ @Override
+ public TopologyResponse startIndexingTopology() throws RestException {
+ return createResponse(stormCLIClientWrapper.startIndexingTopology(), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
+ }
+
+ @Override
+ public TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException {
+ return createResponse(stormCLIClientWrapper.stopIndexingTopology(stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
+ }
+
+ private TopologyResponse createResponse(int responseCode, TopologyStatusCode successMessage, TopologyStatusCode errorMessage) {
+ TopologyResponse topologyResponse = new TopologyResponse();
+ if (responseCode == 0) {
+ topologyResponse.setSuccessMessage(successMessage.toString());
+ } else {
+ topologyResponse.setErrorMessage(errorMessage.toString());
+ }
+ return topologyResponse;
+ }
+
+ @Override
+ public Map<String, String> getStormClientStatus() throws RestException {
+ return stormCLIClientWrapper.getStormClientStatus();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
new file mode 100644
index 0000000..a472515
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.metron.rest.MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME;
+import static org.apache.metron.rest.MetronRestConstants.INDEXING_TOPOLOGY_NAME;
+
+public class StormCLIWrapper {
+
+ @Autowired
+ private Environment environment;
+
+ public int startParserTopology(String name) throws RestException {
+ return runCommand(getParserStartCommand(name));
+ }
+
+ public int stopParserTopology(String name, boolean stopNow) throws RestException {
+ return runCommand(getStopCommand(name, stopNow));
+ }
+
+ public int startEnrichmentTopology() throws RestException {
+ return runCommand(getEnrichmentStartCommand());
+ }
+
+ public int stopEnrichmentTopology(boolean stopNow) throws RestException {
+ return runCommand(getStopCommand(ENRICHMENT_TOPOLOGY_NAME, stopNow));
+ }
+
+ public int startIndexingTopology() throws RestException {
+ return runCommand(getIndexingStartCommand());
+ }
+
+ public int stopIndexingTopology(boolean stopNow) throws RestException {
+ return runCommand(getStopCommand(INDEXING_TOPOLOGY_NAME, stopNow));
+ }
+
+ protected int runCommand(String[] command) throws RestException {
+ ProcessBuilder pb = getProcessBuilder(command);
+ pb.inheritIO();
+ Process process = null;
+ try {
+ process = pb.start();
+ process.waitFor();
+ } catch (Exception e) {
+ throw new RestException(e);
+ }
+ return process.exitValue();
+ }
+
+ protected String[] getParserStartCommand(String name) {
+ String[] command = new String[7];
+ command[0] = environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY);
+ command[1] = "-k";
+ command[2] = environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY);
+ command[3] = "-z";
+ command[4] = environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY);
+ command[5] = "-s";
+ command[6] = name;
+ return command;
+ }
+
+ protected String[] getEnrichmentStartCommand() {
+ String[] command = new String[1];
+ command[0] = environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY);
+ return command;
+ }
+
+ protected String[] getIndexingStartCommand() {
+ String[] command = new String[1];
+ command[0] = environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY);
+ return command;
+ }
+
+ protected String[] getStopCommand(String name, boolean stopNow) {
+ String[] command;
+ if (stopNow) {
+ command = new String[5];
+ command[3] = "-w";
+ command[4] = "0";
+ } else {
+ command = new String[3];
+ }
+ command[0] = "storm";
+ command[1] = "kill";
+ command[2] = name;
+ return command;
+ }
+
+ protected ProcessBuilder getProcessBuilder(String... command) {
+ return new ProcessBuilder(command);
+ }
+
+ public Map<String, String> getStormClientStatus() throws RestException {
+ Map<String, String> status = new HashMap<>();
+ status.put("parserScriptPath", environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY));
+ status.put("enrichmentScriptPath", environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY));
+ status.put("indexingScriptPath", environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY));
+ status.put("stormClientVersionInstalled", stormClientVersionInstalled());
+ return status;
+ }
+
+ protected String stormClientVersionInstalled() throws RestException {
+ String stormClientVersionInstalled = "Storm client is not installed";
+ ProcessBuilder pb = getProcessBuilder("storm", "version");
+ pb.redirectErrorStream(true);
+ Process p;
+ try {
+ p = pb.start();
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ List<String> lines = reader.lines().collect(toList());
+ lines.forEach(System.out::println);
+ if (lines.size() > 1) {
+ stormClientVersionInstalled = lines.get(1).replaceFirst("Storm ", "");
+ }
+ return stormClientVersionInstalled;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
new file mode 100644
index 0000000..280d72a
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatus;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.model.TopologySummary;
+import org.apache.metron.rest.service.StormStatusService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.metron.rest.MetronRestConstants.STORM_UI_SPRING_PROPERTY;
+import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_SUMMARY_URL;
+import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL;
+
+@Service
+public class StormStatusServiceImpl implements StormStatusService {
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private RestTemplate restTemplate;
+
+ @Override
+ public TopologySummary getTopologySummary() {
+ return restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_SUMMARY_URL, TopologySummary.class);
+ }
+
+ @Override
+ public TopologyStatus getTopologyStatus(String name) {
+ TopologyStatus topologyResponse = null;
+ String id = null;
+ for (TopologyStatus topology : getTopologySummary().getTopologies()) {
+ if (name.equals(topology.getName())) {
+ id = topology.getId();
+ break;
+ }
+ }
+ if (id != null) {
+ topologyResponse = restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id, TopologyStatus.class);
+ }
+ return topologyResponse;
+ }
+
+ @Override
+ public List<TopologyStatus> getAllTopologyStatus() {
+ List<TopologyStatus> topologyStatus = new ArrayList<>();
+ for (TopologyStatus topology : getTopologySummary().getTopologies()) {
+ topologyStatus.add(restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + topology.getId(), TopologyStatus.class));
+ }
+ return topologyStatus;
+ }
+
+ @Override
+ public TopologyResponse activateTopology(String name) {
+ TopologyResponse topologyResponse = new TopologyResponse();
+ String id = null;
+ for (TopologyStatus topology : getTopologySummary().getTopologies()) {
+ if (name.equals(topology.getName())) {
+ id = topology.getId();
+ break;
+ }
+ }
+ if (id != null) {
+ Map result = restTemplate.postForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id + "/activate", null, Map.class);
+ if("success".equals(result.get("status"))) {
+ topologyResponse.setSuccessMessage(TopologyStatusCode.ACTIVE.toString());
+ } else {
+ topologyResponse.setErrorMessage((String) result.get("status"));
+ }
+ } else {
+ topologyResponse.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString());
+ }
+ return topologyResponse;
+ }
+
+ @Override
+ public TopologyResponse deactivateTopology(String name) {
+ TopologyResponse topologyResponse = new TopologyResponse();
+ String id = null;
+ for (TopologyStatus topology : getTopologySummary().getTopologies()) {
+ if (name.equals(topology.getName())) {
+ id = topology.getId();
+ break;
+ }
+ }
+ if (id != null) {
+ Map result = restTemplate.postForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id + "/deactivate", null, Map.class);
+ if("success".equals(result.get("status"))) {
+ topologyResponse.setSuccessMessage(TopologyStatusCode.INACTIVE.toString());
+ } else {
+ topologyResponse.setErrorMessage((String) result.get("status"));
+ }
+ } else {
+ topologyResponse.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString());
+ }
+ return topologyResponse;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
new file mode 100644
index 0000000..b26dc82
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctionInfo;
+import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.TransformationValidation;
+import org.apache.metron.rest.service.TransformationService;
+import org.json.simple.JSONObject;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+public class TransformationServiceImpl implements TransformationService {
+
+ @Override
+ public Map<String, Boolean> validateRules(List<String> rules) {
+ Map<String, Boolean> results = new HashMap<>();
+ StellarProcessor stellarProcessor = new StellarProcessor();
+ for(String rule: rules) {
+ try {
+ boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
+ results.put(rule, result);
+ } catch (ParseException e) {
+ results.put(rule, false);
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public Map<String, Object> validateTransformation(TransformationValidation transformationValidation) {
+ JSONObject sampleJson = new JSONObject(transformationValidation.getSampleData());
+ transformationValidation.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
+ fieldTransformer.transformAndUpdate(sampleJson, transformationValidation.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
+ }
+ );
+ return sampleJson;
+ }
+
+ @Override
+ public FieldTransformations[] getTransformations() {
+ return FieldTransformations.values();
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
+ Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
+ stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
+ stellarFunctionDescriptions.add(new StellarFunctionDescription(
+ stellarFunctionInfo.getName(),
+ stellarFunctionInfo.getDescription(),
+ stellarFunctionInfo.getParams(),
+ stellarFunctionInfo.getReturns()));
+ });
+ return stellarFunctionDescriptions;
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getSimpleStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
+ return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
+ stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
+ }
+
+}
[07/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
METRON-503: Metron REST API this closes apache/incubator-metron#316
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/a61dbcf7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/a61dbcf7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/a61dbcf7
Branch: refs/heads/master
Commit: a61dbcf7ed69768b2342b292460815e5eee0db07
Parents: 55b3e7e
Author: JJ <jj...@gmail.com>
Authored: Sun Jan 29 19:31:37 2017 -0600
Committer: cstella <ce...@gmail.com>
Committed: Thu Mar 2 10:56:56 2017 -0500
----------------------------------------------------------------------
.../metron/rest/model/GrokValidation.java | 20 ++
.../apache/metron/rest/model/KafkaTopic.java | 22 ++
.../apache/metron/rest/config/KafkaConfig.java | 8 +-
.../apache/metron/rest/config/StormConfig.java | 2 +-
.../metron/rest/controller/GrokController.java | 2 +-
.../metron/rest/service/KafkaService.java | 2 +-
.../service/impl/DockerStormCLIWrapper.java | 30 +-
.../service/impl/GlobalConfigServiceImpl.java | 5 +-
.../rest/service/impl/GrokServiceImpl.java | 14 +-
.../rest/service/impl/KafkaServiceImpl.java | 52 ++--
.../apache/metron/rest/config/TestConfig.java | 8 +-
.../GrokControllerIntegrationTest.java | 2 +-
.../metron/rest/service/HdfsServiceTest.java | 117 --------
.../service/impl/DockerStormCLIWrapperTest.java | 83 ++++++
.../impl/GlobalConfigServiceImplTest.java | 154 ++++++++++
.../rest/service/impl/GrokServiceImplTest.java | 151 ++++++++++
.../rest/service/impl/HdfsServiceImplTest.java | 117 ++++++++
.../rest/service/impl/KafkaServiceImplTest.java | 292 +++++++++++++++++++
18 files changed, 913 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
index 7fbcd34..ccd2c5c 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
@@ -52,4 +52,24 @@ public class GrokValidation {
public void setResults(Map<String, Object> results) {
this.results = results;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ GrokValidation that = (GrokValidation) o;
+
+ if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false;
+ if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false;
+ return results != null ? results.equals(that.results) : that.results == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = statement != null ? statement.hashCode() : 0;
+ result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0);
+ result = 31 * result + (results != null ? results.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
index c8db9f6..55dd2b2 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java
@@ -57,4 +57,26 @@ public class KafkaTopic {
public void setProperties(Properties properties) {
this.properties = properties;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ KafkaTopic that = (KafkaTopic) o;
+
+ if (numPartitions != that.numPartitions) return false;
+ if (replicationFactor != that.replicationFactor) return false;
+ if (name != null ? !name.equals(that.name) : that.name != null) return false;
+ return properties != null ? properties.equals(that.properties) : that.properties == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + numPartitions;
+ result = 31 * result + replicationFactor;
+ result = 31 * result + (properties != null ? properties.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
index 8044405..f6ff73c 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.config;
+import kafka.admin.AdminUtils$;
import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -46,7 +47,7 @@ public class KafkaConfig {
return ZkUtils.apply(zkClient, false);
}
- @Bean(destroyMethod="close")
+ @Bean(destroyMethod = "close")
public KafkaConsumer<String, String> kafkaConsumer() {
Properties props = new Properties();
props.put("bootstrap.servers", environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY));
@@ -58,4 +59,9 @@ public class KafkaConfig {
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
return new KafkaConsumer<>(props);
}
+
+ @Bean
+ public AdminUtils$ adminUtils() {
+ return AdminUtils$.MODULE$;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
index d6d0cff..7a61cbc 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
@@ -40,7 +40,7 @@ public class StormConfig {
@Bean
public StormCLIWrapper stormCLIClientWrapper() {
if (Arrays.asList(environment.getActiveProfiles()).contains(DOCKER_PROFILE)) {
- return new DockerStormCLIWrapper();
+ return new DockerStormCLIWrapper(environment);
} else {
return new StormCLIWrapper();
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
index 2b155b1..d561897 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
@@ -43,7 +43,7 @@ public class GrokController {
@ApiOperation(value = "Applies a Grok statement to a sample message")
@ApiResponse(message = "JSON results", code = 200)
@RequestMapping(value = "/validate", method = RequestMethod.POST)
- ResponseEntity<GrokValidation> post(@ApiParam(name="grokValidation", value="Object containing Grok statment and sample message", required=true)@RequestBody GrokValidation grokValidation) throws RestException {
+ ResponseEntity<GrokValidation> post(@ApiParam(name = "grokValidation", value = "Object containing Grok statement and sample message", required = true) @RequestBody GrokValidation grokValidation) throws RestException {
return new ResponseEntity<>(grokService.validateGrokStatement(grokValidation), HttpStatus.OK);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
index e89b165..f3cd901 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
@@ -23,6 +23,7 @@ import org.apache.metron.rest.model.KafkaTopic;
import java.util.Set;
public interface KafkaService {
+ String CONSUMER_OFFSETS_TOPIC = "__consumer_offsets";
KafkaTopic createTopic(KafkaTopic topic) throws RestException;
@@ -33,5 +34,4 @@ public interface KafkaService {
Set<String> listTopics();
String getSampleMessage(String topic);
-
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
index d7bbb23..059afd7 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java
@@ -30,30 +30,34 @@ import java.util.Map;
public class DockerStormCLIWrapper extends StormCLIWrapper {
- private Logger LOG = LoggerFactory.getLogger(DockerStormCLIWrapper.class);
+ private static final Logger LOG = LoggerFactory.getLogger(DockerStormCLIWrapper.class);
- @Autowired
private Environment environment;
+ @Autowired
+ public DockerStormCLIWrapper(final Environment environment) {
+ this.environment = environment;
+ }
+
@Override
- protected ProcessBuilder getProcessBuilder(String... command) {
- String[] dockerCommand = {"docker-compose", "-f", environment.getProperty("docker.compose.path"), "-p", "metron", "exec", "storm"};
- ProcessBuilder pb = new ProcessBuilder(ArrayUtils.addAll(dockerCommand, command));
- Map<String, String> pbEnvironment = pb.environment();
+ protected ProcessBuilder getProcessBuilder(final String... command) {
+ final String[] dockerCommand = {"docker-compose", "-f", environment.getProperty("docker.compose.path"), "-p", "metron", "exec", "storm"};
+ final ProcessBuilder pb = new ProcessBuilder(ArrayUtils.addAll(dockerCommand, command));
+ final Map<String, String> pbEnvironment = pb.environment();
pbEnvironment.put("METRON_VERSION", environment.getProperty("metron.version"));
setDockerEnvironment(pbEnvironment);
return pb;
}
- protected void setDockerEnvironment(Map<String, String> environmentVariables) {
- ProcessBuilder pb = getDockerEnvironmentProcessBuilder();
+ private void setDockerEnvironment(final Map<String, String> environmentVariables) {
+ final ProcessBuilder pb = getDockerEnvironmentProcessBuilder();
try {
- Process process = pb.start();
- BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream()));
+ final Process process = pb.start();
+ final BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
- while((line = inputStream.readLine()) != null) {
+ while ((line = inputStream.readLine()) != null) {
if (line.startsWith("export")) {
- String[] parts = line.replaceFirst("export ", "").split("=");
+ final String[] parts = line.replaceFirst("export ", "").split("=");
environmentVariables.put(parts[0], parts[1].replaceAll("\"", ""));
}
}
@@ -63,7 +67,7 @@ public class DockerStormCLIWrapper extends StormCLIWrapper {
}
}
- protected ProcessBuilder getDockerEnvironmentProcessBuilder() {
+ private ProcessBuilder getDockerEnvironmentProcessBuilder() {
String[] command = {"docker-machine", "env", "metron-machine"};
return new ProcessBuilder(command);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
index 54c331a..e80380b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java
@@ -33,9 +33,12 @@ import java.util.Map;
@Service
public class GlobalConfigServiceImpl implements GlobalConfigService {
+ private CuratorFramework client;
@Autowired
- private CuratorFramework client;
+ public GlobalConfigServiceImpl(CuratorFramework client) {
+ this.client = client;
+ }
@Override
public Map<String, Object> save(Map<String, Object> globalConfig) throws RestException {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 8fbea13..323ca78 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.service.impl;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
+import org.apache.directory.api.util.Strings;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
@@ -31,9 +32,12 @@ import java.util.Map;
@Service
public class GrokServiceImpl implements GrokService {
+ private Grok commonGrok;
@Autowired
- private Grok commonGrok;
+ public GrokServiceImpl(Grok commonGrok) {
+ this.commonGrok = commonGrok;
+ }
@Override
public Map<String, String> getCommonGrokPatterns() {
@@ -44,10 +48,12 @@ public class GrokServiceImpl implements GrokService {
public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException {
Map<String, Object> results;
try {
+ String statement = Strings.isEmpty(grokValidation.getStatement()) ? "" : grokValidation.getStatement();
+
Grok grok = new Grok();
grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
- grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
- String patternLabel = grokValidation.getStatement().substring(0, grokValidation.getStatement().indexOf(" "));
+ grok.addPatternFromReader(new StringReader(statement));
+ String patternLabel = statement.substring(0, statement.indexOf(" "));
String grokPattern = "%{" + patternLabel + "}";
grok.compile(grokPattern);
Match gm = grok.match(grokValidation.getSampleData());
@@ -55,7 +61,7 @@ public class GrokServiceImpl implements GrokService {
results = gm.toMap();
results.remove(patternLabel);
} catch (StringIndexOutOfBoundsException e) {
- throw new RestException("A pattern label must be included (ex. PATTERN_LABEL ${PATTERN:field} ...)", e.getCause());
+ throw new RestException("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)", e.getCause());
} catch (Exception e) {
throw new RestException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
index 7246c2f..7b10dc4 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
@@ -18,10 +18,9 @@
package org.apache.metron.rest.service.impl;
import kafka.admin.AdminOperationException;
-import kafka.admin.AdminUtils;
+import kafka.admin.AdminUtils$;
import kafka.admin.RackAwareMode;
import kafka.utils.ZkUtils;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
@@ -32,27 +31,30 @@ import org.apache.metron.rest.service.KafkaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import java.util.Iterator;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
public class KafkaServiceImpl implements KafkaService {
-
- @Autowired
private ZkUtils zkUtils;
-
- @Autowired
private KafkaConsumer<String, String> kafkaConsumer;
+ private AdminUtils$ adminUtils;
- private String offsetTopic = "__consumer_offsets";
+ @Autowired
+ public KafkaServiceImpl(ZkUtils zkUtils, KafkaConsumer<String, String> kafkaConsumer, AdminUtils$ adminUtils) {
+ this.zkUtils = zkUtils;
+ this.kafkaConsumer = kafkaConsumer;
+ this.adminUtils = adminUtils;
+ }
@Override
public KafkaTopic createTopic(KafkaTopic topic) throws RestException {
if (!listTopics().contains(topic.getName())) {
try {
- AdminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$);
+ adminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$);
} catch (AdminOperationException e) {
throw new RestException(e);
}
@@ -62,8 +64,9 @@ public class KafkaServiceImpl implements KafkaService {
@Override
public boolean deleteTopic(String name) {
- if (listTopics().contains(name)) {
- AdminUtils.deleteTopic(zkUtils, name);
+ Set<String> topics = listTopics();
+ if (topics != null && topics.contains(name)) {
+ adminUtils.deleteTopic(zkUtils, name);
return true;
} else {
return false;
@@ -90,8 +93,9 @@ public class KafkaServiceImpl implements KafkaService {
public Set<String> listTopics() {
Set<String> topics;
synchronized (this) {
- topics = kafkaConsumer.listTopics().keySet();
- topics.remove(offsetTopic);
+ Map<String, List<PartitionInfo>> topicsInfo = kafkaConsumer.listTopics();
+ topics = topicsInfo == null ? new HashSet<>() : topicsInfo.keySet();
+ topics.remove(CONSUMER_OFFSETS_TOPIC);
}
return topics;
}
@@ -101,20 +105,16 @@ public class KafkaServiceImpl implements KafkaService {
String message = null;
if (listTopics().contains(topic)) {
synchronized (this) {
- kafkaConsumer.assign(kafkaConsumer.partitionsFor(topic).stream().map(partitionInfo ->
- new TopicPartition(topic, partitionInfo.partition())).collect(Collectors.toList()));
- for (TopicPartition topicPartition : kafkaConsumer.assignment()) {
- long offset = kafkaConsumer.position(topicPartition) - 1;
- if (offset >= 0) {
- kafkaConsumer.seek(topicPartition, offset);
- }
- }
+ kafkaConsumer.assign(kafkaConsumer.partitionsFor(topic).stream()
+ .map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition()))
+ .collect(Collectors.toList()));
+
+ kafkaConsumer.assignment().stream()
+ .filter(p -> (kafkaConsumer.position(p) -1) >= 0)
+ .forEach(p -> kafkaConsumer.seek(p, kafkaConsumer.position(p) - 1));
+
ConsumerRecords<String, String> records = kafkaConsumer.poll(100);
- Iterator<ConsumerRecord<String, String>> iterator = records.iterator();
- if (iterator.hasNext()) {
- ConsumerRecord<String, String> record = iterator.next();
- message = record.value();
- }
+ message = records.isEmpty() ? null : records.iterator().next().value();
kafkaConsumer.unsubscribe();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index 7931fe6..edfd542 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.rest.config;
-import com.google.common.base.Function;
+import kafka.admin.AdminUtils$;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
@@ -38,7 +38,6 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.web.client.RestTemplate;
-import javax.annotation.Nullable;
import java.util.Properties;
import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
@@ -121,4 +120,9 @@ public class TestConfig {
restTemplate.setMockStormCLIClientWrapper((MockStormCLIClientWrapper) stormCLIClientWrapper);
return restTemplate;
}
+
+ @Bean
+ public AdminUtils$ adminUtils() {
+ return AdminUtils$.MODULE$;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
index e618d48..4532616 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -105,7 +105,7 @@ public class GrokControllerIntegrationTest {
.andExpect(status().isInternalServerError())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(jsonPath("$.responseCode").value(500))
- .andExpect(jsonPath("$.message").value("A pattern label must be included (ex. PATTERN_LABEL ${PATTERN:field} ...)"));
+ .andExpect(jsonPath("$.message").value("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)"));
this.mockMvc.perform(get(grokUrl + "/list").with(httpBasic(user,password)))
.andExpect(status().isOk())
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
deleted file mode 100644
index f7e43ab..0000000
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/HdfsServiceTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.metron.rest.config.HadoopConfig;
-import org.apache.metron.rest.service.impl.HdfsServiceImpl;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Profile;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceTest.HdfsServiceTestContextConfiguration.class})
-@ActiveProfiles(TEST_PROFILE)
-public class HdfsServiceTest {
-
- @Configuration
- @Profile("test")
- static class HdfsServiceTestContextConfiguration {
-
- @Bean
- public HdfsService hdfsService() {
- return new HdfsServiceImpl();
- }
- }
-
- @Autowired
- private HdfsService hdfsService;
-
- @Test
- public void test() throws IOException {
- String rootDir = "./src/test/tmp";
- File rootFile = new File(rootDir);
- Path rootPath = new Path(rootDir);
- if (rootFile.exists()) {
- FileUtils.cleanDirectory(rootFile);
- FileUtils.deleteDirectory(rootFile);
- }
- assertEquals(true, rootFile.mkdir());
- String fileName1 = "fileName1";
- String fileName2 = "fileName2";
- Path path1 = new Path(rootDir, fileName1);
- String value1 = "value1";
- String value2 = "value2";
- Path path2 = new Path(rootDir, fileName2);
- String invalidFile = "invalidFile";
- Path pathInvalidFile = new Path(rootDir, invalidFile);
-
- FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir));
- assertEquals(0, fileStatuses.length);
-
-
- hdfsService.write(path1, value1.getBytes());
- assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1)));
- assertEquals(value1, new String(hdfsService.read(path1)));
-
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(1, fileStatuses.length);
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
- hdfsService.write(path2, value2.getBytes());
- assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2)));
- assertEquals(value2, new String(hdfsService.read(path2)));
-
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(2, fileStatuses.length);
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
- assertEquals(true, hdfsService.delete(path1, false));
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(1, fileStatuses.length);
- assertEquals(fileName2, fileStatuses[0].getPath().getName());
- assertEquals(true, hdfsService.delete(path2, false));
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(0, fileStatuses.length);
-
- try {
- hdfsService.read(pathInvalidFile);
- fail("Exception should be thrown when reading invalid file name");
- } catch(IOException e) {
- }
- assertEquals(false, hdfsService.delete(pathInvalidFile, false));
-
- FileUtils.deleteDirectory(new File(rootDir));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
new file mode 100644
index 0000000..1217bcb
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.core.env.Environment;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
+
+@SuppressWarnings("unchecked")
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DockerStormCLIWrapper.class, ProcessBuilder.class})
+public class DockerStormCLIWrapperTest {
+ private ProcessBuilder processBuilder;
+ private Environment environment;
+ private DockerStormCLIWrapper dockerStormCLIWrapper;
+
+ @Before
+ public void setUp() throws Exception {
+ processBuilder = mock(ProcessBuilder.class);
+ environment = mock(Environment.class);
+
+ dockerStormCLIWrapper = new DockerStormCLIWrapper(environment);
+ }
+
+ @Test
+ public void getProcessBuilderShouldProperlyGenerateProcessorBuilder() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.environment()).thenReturn(new HashMap<>());
+ when(processBuilder.command()).thenReturn(new ArrayList<>());
+
+ Process process = mock(Process.class);
+ InputStream inputStream = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return -1;
+ }
+ };
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.getInputStream()).thenReturn(inputStream);
+ when(environment.getProperty("docker.compose.path")).thenReturn("/test");
+ when(environment.getProperty("metron.version")).thenReturn("1");
+
+
+ ProcessBuilder actualBuilder = dockerStormCLIWrapper.getProcessBuilder("oo", "ooo");
+
+ assertEquals(new HashMap<String, String>() {{ put("METRON_VERSION", "1"); }}, actualBuilder.environment());
+ assertEquals(new ArrayList<>(), actualBuilder.command());
+
+ verify(process).waitFor();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
new file mode 100644
index 0000000..59d5957
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.GetDataBuilder;
+import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class GlobalConfigServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ CuratorFramework curatorFramework;
+ GlobalConfigService globalConfigService;
+
+ @Before
+ public void setUp() throws Exception {
+ curatorFramework = mock(CuratorFramework.class);
+ globalConfigService = new GlobalConfigServiceImpl(curatorFramework);
+ }
+
+
+ @Test
+ public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class);
+
+ assertFalse(globalConfigService.delete());
+ }
+
+ @Test
+ public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(Exception.class);
+
+ assertFalse(globalConfigService.delete());
+ }
+
+ @Test
+ public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenReturn(null);
+
+ assertTrue(globalConfigService.delete());
+
+ verify(curatorFramework).delete();
+ }
+
+ @Test
+ public void getShouldProperlyReturnGlobalConfig() throws Exception {
+ final String config = "{\"k\":\"v\"}";
+ final Map<String, Object> configMap = new HashMap<String, Object>() {{
+ put("k", "v");
+ }};
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenReturn(config.getBytes());
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(configMap, globalConfigService.get());
+ }
+
+ @Test
+ public void getShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertNull(globalConfigService.get());
+ }
+
+ @Test
+ public void getShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(Exception.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ globalConfigService.get();
+ }
+
+ @Test
+ public void saveShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenThrow(Exception.class);
+
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ globalConfigService.save(new HashMap<>());
+ }
+
+ @Test
+ public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenReturn(new Stat());
+
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ assertEquals(new HashMap<>(), globalConfigService.save(new HashMap<>()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{}".getBytes()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
new file mode 100644
index 0000000..7fc8748
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import oi.thekraken.grok.api.Grok;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.GrokValidation;
+import org.apache.metron.rest.service.GrokService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class GrokServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private Grok grok;
+ private GrokService grokService;
+
+ @Before
+ public void setUp() throws Exception {
+ grok = mock(Grok.class);
+ grokService = new GrokServiceImpl(grok);
+ }
+
+ @Test
+ public void getCommonGrokPattersShouldCallGrokToGetPatterns() throws Exception {
+ grokService.getCommonGrokPatterns();
+
+ verify(grok).getPatterns();
+ }
+
+ @Test
+ public void getCommonGrokPattersShouldCallGrokToGetPatternsAndNotAlterValue() throws Exception {
+ Map<String, String> patterns = new HashMap<String, String>() {{
+ put("k", "v");
+ put("k1", "v1");
+ }};
+
+ when(grok.getPatterns()).thenReturn(patterns);
+
+ assertEquals(patterns, grokService.getCommonGrokPatterns());
+ }
+
+ @Test
+ public void validateGrokStatementShouldThrowExceptionWithEmptyStringAsStatement() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData("asdf asdf");
+ grokValidation.setStatement("");
+
+ grokService.validateGrokStatement(grokValidation);
+ }
+
+ @Test
+ public void validateGrokStatementShouldThrowExceptionWithNullStringAsStatement() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData("asdf asdf");
+ grokValidation.setStatement(null);
+
+ grokService.validateGrokStatement(grokValidation);
+ }
+
+ @Test
+ public void validateGrokStatementShouldProperlyMatchSampleDataAgainstGivenStatement() throws Exception {
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData("asdf asdf");
+ grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ GrokValidation expectedGrokValidation = new GrokValidation();
+ expectedGrokValidation.setResults(new HashMap<String, Object>() {{ put("word1", "asdf"); put("word2", "asdf"); }});
+ expectedGrokValidation.setSampleData("asdf asdf");
+ expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ }
+
+ @Test
+ public void validateGrokStatementShouldProperlyMatchNothingAgainstEmptyString() throws Exception {
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData("");
+ grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ GrokValidation expectedGrokValidation = new GrokValidation();
+ expectedGrokValidation.setResults(new HashMap<>());
+ expectedGrokValidation.setSampleData("");
+ expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ }
+
+ @Test
+ public void validateGrokStatementShouldProperlyMatchNothingAgainstNullString() throws Exception {
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData(null);
+ grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ GrokValidation expectedGrokValidation = new GrokValidation();
+ expectedGrokValidation.setResults(new HashMap<>());
+ expectedGrokValidation.setSampleData(null);
+ expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+
+ assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ }
+
+ @Test
+ public void invalidGrokStatementShouldThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData(null);
+ grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2");
+
+ grokService.validateGrokStatement(grokValidation);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
new file mode 100644
index 0000000..d67892e
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.config.HadoopConfig;
+import org.apache.metron.rest.service.HdfsService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceImplTest.HdfsServiceTestContextConfiguration.class})
+@ActiveProfiles(TEST_PROFILE)
+public class HdfsServiceImplTest {
+
+ @Configuration
+ @Profile("test")
+ static class HdfsServiceTestContextConfiguration {
+
+ @Bean
+ public HdfsService hdfsService() {
+ return new HdfsServiceImpl();
+ }
+ }
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ @Test
+ public void test() throws IOException {
+ String rootDir = "./src/test/tmp";
+ File rootFile = new File(rootDir);
+ Path rootPath = new Path(rootDir);
+ if (rootFile.exists()) {
+ FileUtils.cleanDirectory(rootFile);
+ FileUtils.deleteDirectory(rootFile);
+ }
+ assertEquals(true, rootFile.mkdir());
+ String fileName1 = "fileName1";
+ String fileName2 = "fileName2";
+ Path path1 = new Path(rootDir, fileName1);
+ String value1 = "value1";
+ String value2 = "value2";
+ Path path2 = new Path(rootDir, fileName2);
+ String invalidFile = "invalidFile";
+ Path pathInvalidFile = new Path(rootDir, invalidFile);
+
+ FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir));
+ assertEquals(0, fileStatuses.length);
+
+
+ hdfsService.write(path1, value1.getBytes());
+ assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1)));
+ assertEquals(value1, new String(hdfsService.read(path1)));
+
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(1, fileStatuses.length);
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+
+ hdfsService.write(path2, value2.getBytes());
+ assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2)));
+ assertEquals(value2, new String(hdfsService.read(path2)));
+
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(2, fileStatuses.length);
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+ assertEquals(fileName1, fileStatuses[0].getPath().getName());
+
+ assertEquals(true, hdfsService.delete(path1, false));
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(1, fileStatuses.length);
+ assertEquals(fileName2, fileStatuses[0].getPath().getName());
+ assertEquals(true, hdfsService.delete(path2, false));
+ fileStatuses = hdfsService.list(rootPath);
+ assertEquals(0, fileStatuses.length);
+
+ try {
+ hdfsService.read(pathInvalidFile);
+ fail("Exception should be thrown when reading invalid file name");
+ } catch(IOException e) {
+ }
+ assertEquals(false, hdfsService.delete(pathInvalidFile, false));
+
+ FileUtils.deleteDirectory(new File(rootDir));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a61dbcf7/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
new file mode 100644
index 0000000..b211ee6
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import kafka.admin.AdminOperationException;
+import kafka.admin.AdminUtils$;
+import kafka.admin.RackAwareMode;
+import kafka.utils.ZkUtils;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.KafkaTopic;
+import org.apache.metron.rest.service.KafkaService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mock;
+
+@SuppressWarnings("unchecked")
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.management.*") // resolve classloader conflict
+@PrepareForTest({AdminUtils$.class})
+public class KafkaServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private ZkUtils zkUtils;
+ private KafkaConsumer<String, String> kafkaConsumer;
+ private AdminUtils$ adminUtils;
+
+ private KafkaService kafkaService;
+
+ private static final KafkaTopic VALID_KAFKA_TOPIC = new KafkaTopic() {{
+ setReplicationFactor(2);
+ setNumPartitions(1);
+ setName("t");
+ setProperties(new Properties());
+ }};
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setUp() throws Exception {
+ zkUtils = mock(ZkUtils.class);
+ kafkaConsumer = mock(KafkaConsumer.class);
+ adminUtils = mock(AdminUtils$.class);
+
+ kafkaService = new KafkaServiceImpl(zkUtils, kafkaConsumer, adminUtils);
+ }
+
+ @Test
+ public void listTopicsHappyPathWithListTopicsReturningNull() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = null;
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ final Set<String> listedTopics = kafkaService.listTopics();
+
+ assertEquals(Sets.newHashSet(), listedTopics);
+
+ verifyZeroInteractions(zkUtils);
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer, zkUtils, adminUtils);
+ }
+
+ @Test
+ public void listTopicsHappyPathWithListTopicsReturningEmptyMap() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ final Set<String> listedTopics = kafkaService.listTopics();
+
+ assertEquals(Sets.newHashSet(), listedTopics);
+
+ verifyZeroInteractions(zkUtils);
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer, zkUtils);
+ }
+
+ @Test
+ public void listTopicsHappyPath() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("topic1", Lists.newArrayList());
+ topics.put("topic2", Lists.newArrayList());
+ topics.put("topic3", Lists.newArrayList());
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ final Set<String> listedTopics = kafkaService.listTopics();
+
+ assertEquals(Sets.newHashSet("topic1", "topic2", "topic3"), listedTopics);
+
+ verifyZeroInteractions(zkUtils);
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer, zkUtils);
+ }
+
+ @Test
+ public void listTopicsShouldProperlyRemoveOffsetTopic() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("topic1", Lists.newArrayList());
+ topics.put("topic2", Lists.newArrayList());
+ topics.put("topic3", Lists.newArrayList());
+ topics.put("__consumer_offsets", Lists.newArrayList());
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ final Set<String> listedTopics = kafkaService.listTopics();
+
+ assertEquals(Sets.newHashSet("topic1", "topic2", "topic3"), listedTopics);
+
+ verifyZeroInteractions(zkUtils);
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer, zkUtils);
+ }
+
+ @Test
+ public void deletingTopicThatDoesNotExistShouldReturnFalse() throws Exception {
+ when(kafkaConsumer.listTopics()).thenReturn(Maps.newHashMap());
+
+ assertFalse(kafkaService.deleteTopic("non_existent_topic"));
+
+ verifyZeroInteractions(zkUtils);
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer, zkUtils);
+ }
+
+ @Test
+ public void deletingTopicThatExistShouldReturnTrue() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("non_existent_topic", Lists.newArrayList());
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ assertTrue(kafkaService.deleteTopic("non_existent_topic"));
+
+ verify(kafkaConsumer).listTopics();
+ verify(adminUtils).deleteTopic(zkUtils, "non_existent_topic");
+ verifyNoMoreInteractions(kafkaConsumer);
+ }
+
+ @Test
+ public void makeSureDeletingTopicReturnsFalseWhenNoTopicsExist() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = null;
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ assertFalse(kafkaService.deleteTopic("non_existent_topic"));
+
+ verify(kafkaConsumer).listTopics();
+ verifyNoMoreInteractions(kafkaConsumer);
+ }
+
+ @Test
+ public void getTopicShouldProperlyMapTopicToKafkaTopic() throws Exception {
+ final PartitionInfo partitionInfo = mock(PartitionInfo.class);
+ when(partitionInfo.replicas()).thenReturn(new Node[] {new Node(1, "host", 8080)});
+
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("t", Lists.newArrayList(partitionInfo));
+ topics.put("t1", Lists.newArrayList());
+
+ final KafkaTopic expected = new KafkaTopic();
+ expected.setName("t");
+ expected.setNumPartitions(1);
+ expected.setReplicationFactor(1);
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+ when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList(partitionInfo));
+
+ assertEquals(expected, kafkaService.getTopic("t"));
+ }
+
+ @Test
+ public void getTopicShouldProperlyHandleTopicsThatDontExist() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("t1", Lists.newArrayList());
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+ when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList());
+
+ assertEquals(null, kafkaService.getTopic("t"));
+
+ verify(kafkaConsumer).listTopics();
+ verify(kafkaConsumer, times(0)).partitionsFor("t");
+ verifyZeroInteractions(zkUtils);
+ verifyNoMoreInteractions(kafkaConsumer);
+ }
+
+ @Test
+ public void createTopicShouldFailIfReplicationFactorIsGreaterThanAvailableBrokers() throws Exception {
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ kafkaService.createTopic(VALID_KAFKA_TOPIC);
+
+ verify(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$));
+ verify(kafkaConsumer).listTopics();
+ verifyZeroInteractions(zkUtils);
+ }
+
+ @Test
+ public void whenAdminUtilsThrowsAdminOperationExceptionCreateTopicShouldProperlyWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ final Map<String, List<PartitionInfo>> topics = new HashMap<>();
+ topics.put("1", new ArrayList<>());
+
+ when(kafkaConsumer.listTopics()).thenReturn(topics);
+
+ doThrow(AdminOperationException.class).when(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$));
+
+ kafkaService.createTopic(VALID_KAFKA_TOPIC);
+ }
+
+ @Test
+ public void getSampleMessageProperlyReturnsAMessageFromAGivenKafkaTopic() throws Exception {
+ final String topicName = "t";
+ final Node host = new Node(1, "host", 8080);
+ final Node[] replicas = {host};
+ final List<PartitionInfo> partitionInfo = Lists.newArrayList(new PartitionInfo(topicName, 1, host, replicas, replicas));
+ final TopicPartition topicPartition = new TopicPartition(topicName, 1);
+ final List<TopicPartition> topicPartitions = Lists.newArrayList(topicPartition);
+ final Set<TopicPartition> topicPartitionsSet = Sets.newHashSet(topicPartitions);
+ final ConsumerRecords<String, String> records = new ConsumerRecords<>(new HashMap<TopicPartition, List<ConsumerRecord<String, String>>>() {{
+ put(topicPartition, Lists.newArrayList(new ConsumerRecord<>(topicName, 1, 1, "k", "message")));
+ }});
+
+ when(kafkaConsumer.listTopics()).thenReturn(new HashMap<String, List<PartitionInfo>>() {{ put(topicName, Lists.newArrayList()); }});
+ when(kafkaConsumer.partitionsFor(eq(topicName))).thenReturn(partitionInfo);
+ when(kafkaConsumer.assignment()).thenReturn(topicPartitionsSet);
+ when(kafkaConsumer.position(topicPartition)).thenReturn(1L);
+ when(kafkaConsumer.poll(100)).thenReturn(records);
+
+ assertEquals("message", kafkaService.getSampleMessage(topicName));
+
+ verify(kafkaConsumer).assign(eq(topicPartitions));
+ verify(kafkaConsumer).assignment();
+ verify(kafkaConsumer).poll(100);
+ verify(kafkaConsumer).unsubscribe();
+ verify(kafkaConsumer, times(2)).position(topicPartition);
+ verify(kafkaConsumer).seek(topicPartition, 0);
+
+ verifyZeroInteractions(zkUtils, adminUtils);
+ }
+}
[09/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
index a472515..c5569c5 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -35,9 +35,13 @@ import static org.apache.metron.rest.MetronRestConstants.INDEXING_TOPOLOGY_NAME;
public class StormCLIWrapper {
- @Autowired
private Environment environment;
+ @Autowired
+ public void setEnvironment(final Environment environment) {
+ this.environment = environment;
+ }
+
public int startParserTopology(String name) throws RestException {
return runCommand(getParserStartCommand(name));
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
index 280d72a..a02ee75 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java
@@ -38,12 +38,16 @@ import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL;
@Service
public class StormStatusServiceImpl implements StormStatusService {
- @Autowired
private Environment environment;
- @Autowired
private RestTemplate restTemplate;
+ @Autowired
+ public StormStatusServiceImpl(Environment environment, RestTemplate restTemplate) {
+ this.environment = environment;
+ this.restTemplate = restTemplate;
+ }
+
@Override
public TopologySummary getTopologySummary() {
return restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_SUMMARY_URL, TopologySummary.class);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
deleted file mode 100644
index b26dc82..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/TransformationServiceImpl.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service.impl;
-
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunctionInfo;
-import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.common.stellar.StellarProcessor;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-import org.apache.metron.rest.service.TransformationService;
-import org.json.simple.JSONObject;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-@Service
-public class TransformationServiceImpl implements TransformationService {
-
- @Override
- public Map<String, Boolean> validateRules(List<String> rules) {
- Map<String, Boolean> results = new HashMap<>();
- StellarProcessor stellarProcessor = new StellarProcessor();
- for(String rule: rules) {
- try {
- boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
- results.put(rule, result);
- } catch (ParseException e) {
- results.put(rule, false);
- }
- }
- return results;
- }
-
- @Override
- public Map<String, Object> validateTransformation(TransformationValidation transformationValidation) {
- JSONObject sampleJson = new JSONObject(transformationValidation.getSampleData());
- transformationValidation.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
- fieldTransformer.transformAndUpdate(sampleJson, transformationValidation.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
- }
- );
- return sampleJson;
- }
-
- @Override
- public FieldTransformations[] getTransformations() {
- return FieldTransformations.values();
- }
-
- @Override
- public List<StellarFunctionDescription> getStellarFunctions() {
- List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
- Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
- stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
- stellarFunctionDescriptions.add(new StellarFunctionDescription(
- stellarFunctionInfo.getName(),
- stellarFunctionInfo.getDescription(),
- stellarFunctionInfo.getParams(),
- stellarFunctionInfo.getReturns()));
- });
- return stellarFunctionDescriptions;
- }
-
- @Override
- public List<StellarFunctionDescription> getSimpleStellarFunctions() {
- List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
- return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
- stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
index 859ca5e..5891417 100644
--- a/metron-interface/metron-rest/src/main/resources/application-docker.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -21,11 +21,11 @@ docker:
spring:
datasource:
- driverClassName: com.mysql.jdbc.Driver
- url: jdbc:mysql://${docker.host.address}:3306/metronrest
+ driverClassName: org.h2.Driver
+ url: jdbc:h2:file:./metrondb
username: root
password: root
- platform: mysql
+ platform: h2
jpa:
hibernate:
ddl-auto: update
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
new file mode 100644
index 0000000..76bef56
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+spring:
+ datasource:
+ driverClassName: org.h2.Driver
+ url: jdbc:h2:file:./metrondb
+ username: root
+ password: root
+ platform: h2
+ jpa:
+ hibernate:
+ ddl-auto: update
+
+zookeeper:
+ url: node1:2181
+
+kafka:
+ broker:
+ url: node1:6667
+
+hdfs:
+ namenode:
+ url: hdfs://node1:8020
+
+grok:
+ path:
+ temp: ./patterns/temp
+ default: /apps/metron/patterns
+
+storm:
+ ui:
+ url: node1:8744
+ parser:
+ script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh
+ enrichment:
+ script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
+ indexing:
+ script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 7e45ed9..2bb1462 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
metron:
- version: 0.3.0
+ version: 0.3.1
logging:
level:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/scripts/start.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start.sh b/metron-interface/metron-rest/src/main/scripts/start.sh
deleted file mode 100755
index d83a2fd..0000000
--- a/metron-interface/metron-rest/src/main/scripts/start.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-METRON_VERSION=0.3.0
-SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ "$#" -ne 1 ]; then
- echo "Usage: start.sh <path to application.yml file>"
- echo "Path can be absolute or relative to $SCRIPTS_ROOT"
-else
- if [ -z "$MYSQL_CLIENT_HOME" ]; then
- echo MYSQL_CLIENT_HOME is not set
- else
- if [ -z "$HIBERNATE_HOME" ]; then
- echo HIBERNATE_HOME is not set
- else
- java -Dloader.path=$MYSQL_CLIENT_HOME/*,$HIBERNATE_HOME/lib/envers/*,$HIBERNATE_HOME/lib/jpa/*,$HIBERNATE_HOME/lib/required/* -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$1
- fi
- fi
-fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
new file mode 100755
index 0000000..ea5d6c9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+if [ "$#" -lt 1 ]; then
+ echo "Usage: start.sh <path to application.yml file> <spring options>"
+ echo "Path can be absolute or relative to $SCRIPTS_ROOT"
+else
+ java -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$@
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
index 7796c8b..f4e18ea 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.controller;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.rest.service.GlobalConfigService;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -57,6 +58,9 @@ public class GlobalConfigControllerIntegrationTest {
public static String globalJson;
@Autowired
+ private GlobalConfigService globalConfigService;
+
+ @Autowired
private WebApplicationContext wac;
private MockMvc mockMvc;
@@ -84,6 +88,8 @@ public class GlobalConfigControllerIntegrationTest {
@Test
public void test() throws Exception {
+ this.globalConfigService.delete();
+
this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password)))
.andExpect(status().isNotFound());
@@ -103,5 +109,7 @@ public class GlobalConfigControllerIntegrationTest {
this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf()))
.andExpect(status().isNotFound());
+
+ this.globalConfigService.delete();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
index 4532616..e0a9c5b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -48,6 +48,7 @@ public class GrokControllerIntegrationTest {
/**
{
"sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+ "patternLabel":"SQUID",
"statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}"
}
*/
@@ -57,11 +58,20 @@ public class GrokControllerIntegrationTest {
/**
{
"sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
- "statement":""
+ "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}"
+ }
+ */
+ @Multiline
+ public static String missingPatternLabelGrokValidationJson;
+
+ /**
+ {
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+ "patternLabel":"SQUID"
}
*/
@Multiline
- public static String badGrokValidationJson;
+ public static String missingStatementGrokValidationJson;
@Autowired
private WebApplicationContext wac;
@@ -101,11 +111,17 @@ public class GrokControllerIntegrationTest {
.andExpect(jsonPath("$.results.timestamp").value("1467011157.401"))
.andExpect(jsonPath("$.results.url").value("http://www.aliexpress.com/af/shoes.html?"));
- this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(badGrokValidationJson))
+ this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingPatternLabelGrokValidationJson))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("Pattern label is required"));
+
+ this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingStatementGrokValidationJson))
.andExpect(status().isInternalServerError())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(jsonPath("$.responseCode").value(500))
- .andExpect(jsonPath("$.message").value("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)"));
+ .andExpect(jsonPath("$.message").value("Grok statement is required"));
this.mockMvc.perform(get(grokUrl + "/list").with(httpBasic(user,password)))
.andExpect(status().isOk())
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
new file mode 100644
index 0000000..a6c3469
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.service.HdfsService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class HdfsControllerIntegrationTest {
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String hdfsUrl = "/api/v1/hdfs";
+ private String user = "user";
+ private String password = "password";
+ private String path = "./target/hdfsTest.txt";
+ private String fileContents = "file contents";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(hdfsUrl).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(hdfsUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(hdfsUrl).with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.hdfsService.delete(new Path(path), false);
+
+ this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(post(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
+ .andExpect(content().bytes(fileContents.getBytes()));
+
+ this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
index a659772..6a67473 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
@@ -94,9 +94,12 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
]
},
"triageConfig": {
- "riskLevelRules": {
- "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'": 10
- },
+ "riskLevelRules": [
+ {
+ "rule": "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'",
+ "score": 10
+ }
+ ],
"aggregator": "MAX"
}
}
@@ -160,7 +163,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
.andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
- .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
.andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
@@ -177,7 +181,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
.andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
- .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
.andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)))
@@ -194,7 +199,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
.andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
.andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
- .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10))
.andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
@@ -211,7 +217,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
"@.broTest.threatIntel.fieldMap.hbaseThreatIntel[1] == 'ip_dst_addr' &&" +
"@.broTest.threatIntel.fieldToTypeMap.ip_src_addr[0] == 'malicious_ip' &&" +
"@.broTest.threatIntel.fieldToTypeMap.ip_dst_addr[0] == 'malicious_ip' &&" +
- "@.broTest.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"] == 10 &&" +
+ "@.broTest.threatIntel.triageConfig.riskLevelRules[0].rule == \"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\" &&" +
+ "@.broTest.threatIntel.triageConfig.riskLevelRules[0].score == 10 &&" +
"@.broTest.threatIntel.triageConfig.aggregator == 'MAX'" +
")]").exists());
@@ -235,6 +242,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
.andExpect(jsonPath("$[0]").value("geo"))
.andExpect(jsonPath("$[1]").value("host"))
.andExpect(jsonPath("$[2]").value("whois"));
+
+ sensorEnrichmentConfigService.delete("broTest");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index 10edadc..cebcde6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -135,6 +135,8 @@ public class SensorIndexingConfigControllerIntegrationTest {
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+
+ sensorIndexingConfigService.delete("broTest");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index 09b1b0a..45ee1f5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -20,6 +20,8 @@ package org.apache.metron.rest.controller;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.SensorParserConfigService;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -58,7 +60,7 @@ public class SensorParserConfigControllerIntegrationTest {
"sensorTopic": "squidTest",
"parserConfig": {
"patternLabel": "SQUIDTEST",
- "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+ "grokPath": "target/patterns/squidTest",
"timestampField": "timestamp"
},
"fieldTransformations" : [
@@ -78,19 +80,6 @@ public class SensorParserConfigControllerIntegrationTest {
/**
{
- "parserClassName": "org.apache.metron.parsers.GrokParser",
- "sensorTopic": "squidTest",
- "parserConfig": {
- "patternLabel": "SQUIDTEST",
- "timestampField": "timestamp"
- }
- }
- */
- @Multiline
- public static String missingGrokJson;
-
- /**
- {
"parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
"sensorTopic":"broTest",
"parserConfig": {}
@@ -106,12 +95,12 @@ public class SensorParserConfigControllerIntegrationTest {
"parserClassName": "org.apache.metron.parsers.GrokParser",
"sensorTopic": "squidTest",
"parserConfig": {
- "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
- "patternLabel": "SQUIDTEST",
+ "patternLabel": "SQUID_DELIMITED",
"grokPath":"./squidTest",
"timestampField": "timestamp"
}
},
+ "grokStatement":"SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?",
"sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
}
*/
@@ -168,6 +157,9 @@ public class SensorParserConfigControllerIntegrationTest {
private Environment environment;
@Autowired
+ private SensorParserConfigService sensorParserConfigService;
+
+ @Autowired
private WebApplicationContext wac;
private MockMvc mockMvc;
@@ -199,6 +191,8 @@ public class SensorParserConfigControllerIntegrationTest {
@Test
public void test() throws Exception {
cleanFileSystem();
+ this.sensorParserConfigService.delete("broTest");
+ this.sensorParserConfigService.delete("squidTest");
this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
.andExpect(status().isCreated())
@@ -207,7 +201,6 @@ public class SensorParserConfigControllerIntegrationTest {
.andExpect(jsonPath("$.sensorTopic").value("squidTest"))
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
.andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
- .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
.andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
.andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
.andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
@@ -215,12 +208,6 @@ public class SensorParserConfigControllerIntegrationTest {
.andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)"))
.andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"));
- this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingGrokJson))
- .andExpect(status().isInternalServerError())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.responseCode").value(500))
- .andExpect(jsonPath("$.message").value("A grokStatement must be provided"));
-
this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
@@ -228,7 +215,6 @@ public class SensorParserConfigControllerIntegrationTest {
.andExpect(jsonPath("$.sensorTopic").value("squidTest"))
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
.andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
- .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
.andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
.andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
.andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
@@ -350,6 +336,8 @@ public class SensorParserConfigControllerIntegrationTest {
.andExpect(jsonPath("$.responseCode").value(500))
.andExpect(jsonPath("$.message").value("java.lang.ClassNotFoundException: badClass"));
+ this.sensorParserConfigService.delete("broTest");
+ this.sensorParserConfigService.delete("squidTest");
}
private void cleanFileSystem() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
new file mode 100644
index 0000000..7a57173
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class StellarControllerIntegrationTest {
+
+ private String valid = "TO_LOWER(test)";
+ private String invalid = "BAD_FUNCTION(test)";
+ private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]";
+
+ /**
+ {
+ "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]},
+ "sampleData": {"url": "https://caseystella.com/blog"}
+ }
+ */
+ @Multiline
+ public static String sensorParseContext;
+
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String stellarUrl = "/api/v1/stellar";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stellarUrl + "/list"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stellarUrl + "/list/functions"))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE))
+ .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE));
+
+ this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.url").value("https://caseystella.com/blog"))
+ .andExpect(jsonPath("$.url_host").value("caseystella.com"));
+
+ this.mockMvc.perform(get(stellarUrl + "/list").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+ this.mockMvc.perform(get(stellarUrl + "/list/functions").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+ this.mockMvc.perform(get(stellarUrl + "/list/simple/functions").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
index c592159..69d8da6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
@@ -316,5 +316,8 @@ public class StormControllerIntegrationTest {
.andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh"))
.andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh"))
.andExpect(jsonPath("$.indexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh"));
+
+ globalConfigService.delete();
+ sensorParserConfigService.delete("broTest");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
deleted file mode 100644
index ec987e0..0000000
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.controller;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.http.MediaType;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.junit4.SpringRunner;
-import org.springframework.test.web.servlet.MockMvc;
-import org.springframework.test.web.servlet.setup.MockMvcBuilders;
-import org.springframework.web.context.WebApplicationContext;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
-@ActiveProfiles(TEST_PROFILE)
-public class TransformationControllerIntegrationTest {
-
- private String valid = "TO_LOWER(test)";
- private String invalid = "BAD_FUNCTION(test)";
- private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]";
-
- /**
- {
- "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]},
- "sampleData": {"url": "https://caseystella.com/blog"}
- }
- */
- @Multiline
- public static String sensorParseConfigJson;
-
-
- @Autowired
- private WebApplicationContext wac;
-
- private MockMvc mockMvc;
-
- private String transformationUrl = "/api/v1/transformation";
- private String user = "user";
- private String password = "password";
-
- @Before
- public void setup() throws Exception {
- this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
- }
-
- @Test
- public void testSecurity() throws Exception {
- this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
- .andExpect(status().isUnauthorized());
-
- this.mockMvc.perform(post(transformationUrl + "/validate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
- .andExpect(status().isUnauthorized());
-
- this.mockMvc.perform(get(transformationUrl + "/list"))
- .andExpect(status().isUnauthorized());
-
- this.mockMvc.perform(get(transformationUrl + "/list/functions"))
- .andExpect(status().isUnauthorized());
- }
-
- @Test
- public void test() throws Exception {
- this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE))
- .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE));
-
- this.mockMvc.perform(post(transformationUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.url").value("https://caseystella.com/blog"))
- .andExpect(jsonPath("$.url_host").value("caseystella.com"));
-
- this.mockMvc.perform(get(transformationUrl + "/list").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$", hasSize(greaterThan(0))));
-
- this.mockMvc.perform(get(transformationUrl + "/list/functions").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$", hasSize(greaterThan(0))));
-
- this.mockMvc.perform(get(transformationUrl + "/list/simple/functions").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$", hasSize(greaterThan(0))));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
deleted file mode 100644
index 561548e..0000000
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/SensorParserConfigTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.api.DeleteBuilder;
-import org.apache.curator.framework.api.GetChildrenBuilder;
-import org.apache.metron.common.configuration.ConfigurationType;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.rest.service.impl.SensorParserConfigServiceImpl;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.verifyStatic;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ConfigurationsUtils.class})
-public class SensorParserConfigTest {
-
- @Mock
- private GetChildrenBuilder getChildrenBuilder;
-
- @Mock
- private DeleteBuilder deleteBuilder;
-
- @Mock
- private ObjectMapper objectMapper;
-
- @Mock
- private CuratorFramework client;
-
- @Mock
- private GrokService grokService;
-
- @InjectMocks
- private SensorParserConfigServiceImpl sensorParserConfigService;
-
- @Before
- public void setUp() throws Exception {
- MockitoAnnotations.initMocks(this);
- Mockito.when(client.getChildren()).thenReturn(getChildrenBuilder);
- Mockito.when(client.delete()).thenReturn(deleteBuilder);
-
- }
-
- @Test
- public void test() throws Exception {
- mockStatic(ConfigurationsUtils.class);
- SensorParserConfig broParserConfig = new SensorParserConfig();
- broParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
- broParserConfig.setSensorTopic("broTest");
- Mockito.when(objectMapper.writeValueAsString(broParserConfig)).thenReturn(new String(JSONUtils.INSTANCE.toJSON(broParserConfig)));
- sensorParserConfigService.save(broParserConfig);
- verifyStatic(times(1));
- ConfigurationsUtils.writeSensorParserConfigToZookeeper("broTest", JSONUtils.INSTANCE.toJSON(broParserConfig), client);
-
- PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("broTest", client)).thenReturn(broParserConfig);
- assertEquals(broParserConfig, sensorParserConfigService.findOne("broTest"));
-
- SensorParserConfig squidParserConfig = new SensorParserConfig();
- squidParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser");
- squidParserConfig.setSensorTopic("squid");
- PowerMockito.when(ConfigurationsUtils.readSensorParserConfigFromZookeeper("squidTest", client)).thenReturn(squidParserConfig);
-
- List<String> allTypes = new ArrayList<String>() {{
- add("broTest");
- add("squidTest");
- }};
- Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenReturn(allTypes);
- assertEquals(new ArrayList<SensorParserConfig>() {{ add(broParserConfig); add(squidParserConfig); }}, sensorParserConfigService.getAll());
-
- Mockito.when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(new KeeperException.NoNodeException());
- assertEquals(new ArrayList<>(), sensorParserConfigService.getAll());
-
- assertTrue(sensorParserConfigService.delete("broTest"));
- verify(deleteBuilder, times(1)).forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest");
- Mockito.when(deleteBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/broTest")).thenThrow(new KeeperException.NoNodeException());
- assertFalse(sensorParserConfigService.delete("broTest"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
index 1217bcb..bc91b91 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java
@@ -24,7 +24,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.core.env.Environment;
-import java.io.IOException;
+import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
@@ -60,12 +60,7 @@ public class DockerStormCLIWrapperTest {
when(processBuilder.command()).thenReturn(new ArrayList<>());
Process process = mock(Process.class);
- InputStream inputStream = new InputStream() {
- @Override
- public int read() throws IOException {
- return -1;
- }
- };
+ InputStream inputStream = new ByteArrayInputStream("export DOCKER_HOST=\"tcp://192.168.99.100:2376\"".getBytes());
when(processBuilder.start()).thenReturn(process);
when(process.getInputStream()).thenReturn(inputStream);
@@ -75,7 +70,10 @@ public class DockerStormCLIWrapperTest {
ProcessBuilder actualBuilder = dockerStormCLIWrapper.getProcessBuilder("oo", "ooo");
- assertEquals(new HashMap<String, String>() {{ put("METRON_VERSION", "1"); }}, actualBuilder.environment());
+ assertEquals(new HashMap<String, String>() {{
+ put("METRON_VERSION", "1");
+ put("DOCKER_HOST", "tcp://192.168.99.100:2376");
+ }}, actualBuilder.environment());
assertEquals(new ArrayList<>(), actualBuilder.command());
verify(process).waitFor();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index 7fc8748..6dd95c5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import oi.thekraken.grok.api.Grok;
+import org.apache.commons.io.FileUtils;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
@@ -25,26 +26,42 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({GrokServiceImpl.class, FileWriter.class})
public class GrokServiceImplTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
+ private Environment environment;
private Grok grok;
private GrokService grokService;
@Before
public void setUp() throws Exception {
+ environment = mock(Environment.class);
grok = mock(Grok.class);
- grokService = new GrokServiceImpl(grok);
+ grokService = new GrokServiceImpl(environment, grok);
}
@Test
@@ -56,24 +73,42 @@ public class GrokServiceImplTest {
@Test
public void getCommonGrokPattersShouldCallGrokToGetPatternsAndNotAlterValue() throws Exception {
- Map<String, String> patterns = new HashMap<String, String>() {{
+ final Map<String, String> actual = new HashMap<String, String>() {{
put("k", "v");
put("k1", "v1");
}};
- when(grok.getPatterns()).thenReturn(patterns);
+ when(grok.getPatterns()).thenReturn(actual);
+
+ Map<String, String> expected = new HashMap<String, String>() {{
+ put("k", "v");
+ put("k1", "v1");
+ }};
+ assertEquals(expected, grokService.getCommonGrokPatterns());
+ }
+
+ @Test
+ public void validateGrokStatementShouldThrowExceptionWithNullStringAsPatternLabel() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("Pattern label is required");
+
+ GrokValidation grokValidation = new GrokValidation();
+ grokValidation.setResults(new HashMap<>());
+ grokValidation.setSampleData("asdf asdf");
+ grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
- assertEquals(patterns, grokService.getCommonGrokPatterns());
+ grokService.validateGrokStatement(grokValidation);
}
@Test
public void validateGrokStatementShouldThrowExceptionWithEmptyStringAsStatement() throws Exception {
exception.expect(RestException.class);
- exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+ exception.expectMessage("Grok statement is required");
GrokValidation grokValidation = new GrokValidation();
grokValidation.setResults(new HashMap<>());
grokValidation.setSampleData("asdf asdf");
+ grokValidation.setPatternLabel("LABEL");
grokValidation.setStatement("");
grokService.validateGrokStatement(grokValidation);
@@ -82,11 +117,12 @@ public class GrokServiceImplTest {
@Test
public void validateGrokStatementShouldThrowExceptionWithNullStringAsStatement() throws Exception {
exception.expect(RestException.class);
- exception.expectMessage("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)");
+ exception.expectMessage("Grok statement is required");
GrokValidation grokValidation = new GrokValidation();
grokValidation.setResults(new HashMap<>());
grokValidation.setSampleData("asdf asdf");
+ grokValidation.setPatternLabel("LABEL");
grokValidation.setStatement(null);
grokService.validateGrokStatement(grokValidation);
@@ -94,47 +130,55 @@ public class GrokServiceImplTest {
@Test
public void validateGrokStatementShouldProperlyMatchSampleDataAgainstGivenStatement() throws Exception {
- GrokValidation grokValidation = new GrokValidation();
+ final GrokValidation grokValidation = new GrokValidation();
grokValidation.setResults(new HashMap<>());
grokValidation.setSampleData("asdf asdf");
grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ grokValidation.setPatternLabel("LABEL");
- GrokValidation expectedGrokValidation = new GrokValidation();
- expectedGrokValidation.setResults(new HashMap<String, Object>() {{ put("word1", "asdf"); put("word2", "asdf"); }});
- expectedGrokValidation.setSampleData("asdf asdf");
- expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ GrokValidation expected = new GrokValidation();
+ expected.setResults(new HashMap<String, Object>() {{ put("word1", "asdf"); put("word2", "asdf"); }});
+ expected.setSampleData("asdf asdf");
+ expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ expected.setPatternLabel("LABEL");
- assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ GrokValidation actual = grokService.validateGrokStatement(grokValidation);
+ assertEquals(expected, actual);
+ assertEquals(expected.hashCode(), actual.hashCode());
}
@Test
public void validateGrokStatementShouldProperlyMatchNothingAgainstEmptyString() throws Exception {
- GrokValidation grokValidation = new GrokValidation();
+ final GrokValidation grokValidation = new GrokValidation();
grokValidation.setResults(new HashMap<>());
grokValidation.setSampleData("");
+ grokValidation.setPatternLabel("LABEL");
grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
- GrokValidation expectedGrokValidation = new GrokValidation();
- expectedGrokValidation.setResults(new HashMap<>());
- expectedGrokValidation.setSampleData("");
- expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ GrokValidation expected = new GrokValidation();
+ expected.setResults(new HashMap<>());
+ expected.setSampleData("");
+ expected.setPatternLabel("LABEL");
+ expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
- assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ assertEquals(expected, grokService.validateGrokStatement(grokValidation));
}
@Test
public void validateGrokStatementShouldProperlyMatchNothingAgainstNullString() throws Exception {
- GrokValidation grokValidation = new GrokValidation();
+ final GrokValidation grokValidation = new GrokValidation();
grokValidation.setResults(new HashMap<>());
grokValidation.setSampleData(null);
grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ grokValidation.setPatternLabel("LABEL");
- GrokValidation expectedGrokValidation = new GrokValidation();
- expectedGrokValidation.setResults(new HashMap<>());
- expectedGrokValidation.setSampleData(null);
- expectedGrokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ GrokValidation expected = new GrokValidation();
+ expected.setResults(new HashMap<>());
+ expected.setSampleData(null);
+ expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}");
+ expected.setPatternLabel("LABEL");
- assertEquals(expectedGrokValidation, grokService.validateGrokStatement(grokValidation));
+ assertEquals(expected, grokService.validateGrokStatement(grokValidation));
}
@Test
@@ -148,4 +192,44 @@ public class GrokServiceImplTest {
grokService.validateGrokStatement(grokValidation);
}
+
+ @Test
+ public void saveTemporaryShouldProperlySaveFile() throws Exception {
+ new File("./target/user1").delete();
+ String statement = "grok statement";
+
+ Authentication authentication = mock(Authentication.class);
+ when(authentication.getName()).thenReturn("user1");
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+
+ grokService.saveTemporary(statement, "squid");
+
+ File testFile = new File("./target/user1/squid");
+ assertEquals(statement, FileUtils.readFileToString(testFile));
+ testFile.delete();
+ }
+
+ @Test
+ public void saveTemporaryShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ String statement = "grok statement";
+
+ Authentication authentication = mock(Authentication.class);
+ when(authentication.getName()).thenReturn("user1");
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+ whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException());
+
+ grokService.saveTemporary(statement, "squid");
+ }
+
+ @Test
+ public void missingGrokStatementShouldThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("A grokStatement must be provided");
+
+ grokService.saveTemporary(null, "squid");
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
new file mode 100644
index 0000000..d11f5a4
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.HdfsService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({HdfsServiceImpl.class, FileSystem.class})
+public class HdfsServiceImplExceptionTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private Configuration configuration;
+ private HdfsService hdfsService;
+ private String testDir = "./target/hdfsUnitTest";
+
+ @Before
+ public void setup() throws IOException {
+ configuration = new Configuration();
+ hdfsService = new HdfsServiceImpl(configuration);
+
+ mockStatic(FileSystem.class);
+ }
+
+ @Test
+ public void listShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ FileSystem fileSystem = mock(FileSystem.class);
+ when(FileSystem.get(configuration)).thenReturn(fileSystem);
+ when(fileSystem.listStatus(new Path(testDir))).thenThrow(new IOException());
+
+ hdfsService.list(new Path(testDir));
+ }
+
+ @Test
+ public void readShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ FileSystem fileSystem = mock(FileSystem.class);
+ when(FileSystem.get(configuration)).thenReturn(fileSystem);
+ when(fileSystem.open(new Path(testDir))).thenThrow(new IOException());
+
+ hdfsService.read(new Path(testDir));
+ }
+
+ @Test
+ public void writeShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ FileSystem fileSystem = mock(FileSystem.class);
+ when(FileSystem.get(configuration)).thenReturn(fileSystem);
+ when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException());
+
+ hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8));
+ }
+
+ @Test
+ public void deleteShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ FileSystem fileSystem = mock(FileSystem.class);
+ when(FileSystem.get(configuration)).thenReturn(fileSystem);
+ when(fileSystem.delete(new Path(testDir), false)).thenThrow(new IOException());
+
+ hdfsService.delete(new Path(testDir), false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
index d67892e..9c18dc6 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
@@ -18,100 +18,87 @@
package org.apache.metron.rest.service.impl;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.metron.rest.config.HadoopConfig;
import org.apache.metron.rest.service.HdfsService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Profile;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
+import java.util.List;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(classes={HadoopConfig.class, HdfsServiceImplTest.HdfsServiceTestContextConfiguration.class})
-@ActiveProfiles(TEST_PROFILE)
public class HdfsServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
- @Configuration
- @Profile("test")
- static class HdfsServiceTestContextConfiguration {
-
- @Bean
- public HdfsService hdfsService() {
- return new HdfsServiceImpl();
+ private Configuration configuration;
+ private HdfsService hdfsService;
+ private String testDir = "./target/hdfsUnitTest";
+
+ @Before
+ public void setup() throws IOException {
+ configuration = new Configuration();
+ hdfsService = new HdfsServiceImpl(configuration);
+ File file = new File(testDir);
+ if (!file.exists()) {
+ file.mkdirs();
}
+ FileUtils.cleanDirectory(file);
}
- @Autowired
- private HdfsService hdfsService;
+ @After
+ public void teardown() throws IOException {
+ File file = new File(testDir);
+ FileUtils.cleanDirectory(file);
+ }
@Test
- public void test() throws IOException {
- String rootDir = "./src/test/tmp";
- File rootFile = new File(rootDir);
- Path rootPath = new Path(rootDir);
- if (rootFile.exists()) {
- FileUtils.cleanDirectory(rootFile);
- FileUtils.deleteDirectory(rootFile);
- }
- assertEquals(true, rootFile.mkdir());
- String fileName1 = "fileName1";
- String fileName2 = "fileName2";
- Path path1 = new Path(rootDir, fileName1);
- String value1 = "value1";
- String value2 = "value2";
- Path path2 = new Path(rootDir, fileName2);
- String invalidFile = "invalidFile";
- Path pathInvalidFile = new Path(rootDir, invalidFile);
-
- FileStatus[] fileStatuses = hdfsService.list(new Path(rootDir));
- assertEquals(0, fileStatuses.length);
-
-
- hdfsService.write(path1, value1.getBytes());
- assertEquals(value1, FileUtils.readFileToString(new File(rootDir, fileName1)));
- assertEquals(value1, new String(hdfsService.read(path1)));
-
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(1, fileStatuses.length);
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
- hdfsService.write(path2, value2.getBytes());
- assertEquals(value2, FileUtils.readFileToString(new File(rootDir, fileName2)));
- assertEquals(value2, new String(hdfsService.read(path2)));
-
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(2, fileStatuses.length);
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
- assertEquals(fileName1, fileStatuses[0].getPath().getName());
-
- assertEquals(true, hdfsService.delete(path1, false));
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(1, fileStatuses.length);
- assertEquals(fileName2, fileStatuses[0].getPath().getName());
- assertEquals(true, hdfsService.delete(path2, false));
- fileStatuses = hdfsService.list(rootPath);
- assertEquals(0, fileStatuses.length);
-
- try {
- hdfsService.read(pathInvalidFile);
- fail("Exception should be thrown when reading invalid file name");
- } catch(IOException e) {
- }
- assertEquals(false, hdfsService.delete(pathInvalidFile, false));
+ public void listShouldListFiles() throws Exception {
+ FileUtils.writeStringToFile(new File(testDir, "file1.txt"), "value1");
+ FileUtils.writeStringToFile(new File(testDir, "file2.txt"), "value2");
+
+ List<String> paths = hdfsService.list(new Path(testDir));
+ assertEquals(2, paths.size());
+ assertEquals("file1.txt", paths.get(0));
+ assertEquals("file2.txt", paths.get(1));
+ }
+
+
+ @Test
+ public void readShouldProperlyReadContents() throws Exception {
+ String contents = "contents";
+ FileUtils.writeStringToFile(new File(testDir, "readTest.txt"), contents);
+
+ assertEquals("contents", hdfsService.read(new Path(testDir, "readTest.txt")));
+ }
+
+ @Test
+ public void writeShouldProperlyWriteContents() throws Exception {
+ String contents = "contents";
+ hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8));
+
+ assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt")));
+ }
+
+ @Test
+ public void deleteShouldProperlyDeleteFile() throws Exception {
+ String contents = "contents";
+ FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents);
+
+ List<String> paths = hdfsService.list(new Path(testDir));
+ assertEquals(1, paths.size());
+ assertEquals("deleteTest.txt", paths.get(0));
+
+ hdfsService.delete(new Path(testDir, "deleteTest.txt"), false);
- FileUtils.deleteDirectory(new File(rootDir));
+ paths = hdfsService.list(new Path(testDir));
+ assertEquals(0, paths.size());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
index b211ee6..c7d42b3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -213,7 +213,9 @@ public class KafkaServiceImplTest {
when(kafkaConsumer.listTopics()).thenReturn(topics);
when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList(partitionInfo));
- assertEquals(expected, kafkaService.getTopic("t"));
+ KafkaTopic actual = kafkaService.getTopic("t");
+ assertEquals(expected, actual);
+ assertEquals(expected.hashCode(), actual.hashCode());
}
@Test
[10/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
METRON-503: Metron REST API this closes apache/incubator-metron#316
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e6628499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e6628499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e6628499
Branch: refs/heads/master
Commit: e662849932a74187708b0037042c6b06d4750116
Parents: a61dbcf
Author: merrimanr <me...@gmail.com>
Authored: Thu Mar 2 11:04:06 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Mar 2 15:24:11 2017 -0500
----------------------------------------------------------------------
dependencies_with_url.csv | 3 +
metron-interface/metron-rest-client/pom.xml | 2 +-
.../metron/rest/model/GrokValidation.java | 13 +-
.../metron/rest/model/ParseMessageRequest.java | 9 +
.../metron/rest/model/SensorParserContext.java | 51 +++
.../metron/rest/model/TopologyResponse.java | 18 +
.../metron/rest/model/TopologyStatus.java | 28 +-
.../metron/rest/model/TopologySummary.java | 17 +
.../rest/model/TransformationValidation.java | 51 ---
metron-interface/metron-rest/.gitignore | 1 +
metron-interface/metron-rest/README.md | 222 ++++++++----
metron-interface/metron-rest/pom.xml | 39 +--
.../metron/rest/controller/HdfsController.java | 89 +++++
.../rest/controller/StellarController.java | 80 +++++
.../controller/TransformationController.java | 80 -----
.../apache/metron/rest/service/GrokService.java | 3 +
.../apache/metron/rest/service/HdfsService.java | 13 +-
.../rest/service/SensorParserConfigService.java | 3 +
.../metron/rest/service/StellarService.java | 39 +++
.../rest/service/TransformationService.java | 39 ---
.../rest/service/impl/GrokServiceImpl.java | 59 +++-
.../rest/service/impl/HdfsServiceImpl.java | 55 ++-
.../impl/SensorEnrichmentConfigServiceImpl.java | 8 +-
.../impl/SensorIndexingConfigServiceImpl.java | 8 +-
.../impl/SensorParserConfigServiceImpl.java | 141 +-------
.../rest/service/impl/StellarServiceImpl.java | 92 +++++
.../service/impl/StormAdminServiceImpl.java | 13 +-
.../rest/service/impl/StormCLIWrapper.java | 6 +-
.../service/impl/StormStatusServiceImpl.java | 8 +-
.../service/impl/TransformationServiceImpl.java | 92 -----
.../src/main/resources/application-docker.yml | 6 +-
.../src/main/resources/application-vagrant.yml | 51 +++
.../src/main/resources/application.yml | 2 +-
.../metron-rest/src/main/scripts/start.sh | 33 --
.../src/main/scripts/start_metron_rest.sh | 25 ++
.../GlobalConfigControllerIntegrationTest.java | 8 +
.../GrokControllerIntegrationTest.java | 24 +-
.../HdfsControllerIntegrationTest.java | 101 ++++++
...richmentConfigControllerIntegrationTest.java | 23 +-
...IndexingConfigControllerIntegrationTest.java | 2 +
...orParserConfigControllerIntegrationTest.java | 36 +-
.../StellarControllerIntegrationTest.java | 122 +++++++
.../StormControllerIntegrationTest.java | 3 +
...TransformationControllerIntegrationTest.java | 122 -------
.../rest/service/SensorParserConfigTest.java | 116 -------
.../service/impl/DockerStormCLIWrapperTest.java | 14 +-
.../rest/service/impl/GrokServiceImplTest.java | 136 ++++++--
.../impl/HdfsServiceImplExceptionTest.java | 101 ++++++
.../rest/service/impl/HdfsServiceImplTest.java | 143 ++++----
.../rest/service/impl/KafkaServiceImplTest.java | 4 +-
.../SensorEnrichmentConfigServiceImplTest.java | 256 ++++++++++++++
.../SensorIndexingConfigServiceImplTest.java | 234 +++++++++++++
.../impl/SensorParserConfigServiceImplTest.java | 344 +++++++++++++++++++
.../service/impl/StellarServiceImplTest.java | 92 +++++
.../service/impl/StormAdminServiceImplTest.java | 156 +++++++++
.../rest/service/impl/StormCLIWrapperTest.java | 216 ++++++++++++
.../impl/StormStatusServiceImplTest.java | 221 ++++++++++++
.../metron-rest/src/test/resources/README.vm | 121 +++++--
metron-interface/pom.xml | 2 +-
59 files changed, 3013 insertions(+), 983 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 35acced..1d428c3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -283,3 +283,6 @@ org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,AS
org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org
+com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html
+de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html
+org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
index 66ccf52..0a49986 100644
--- a/metron-interface/metron-rest-client/pom.xml
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.3.0</version>
+ <version>0.3.1</version>
</parent>
<artifactId>metron-rest-client</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
index ccd2c5c..2795320 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
@@ -22,10 +22,19 @@ import java.util.Map;
public class GrokValidation {
+ private String patternLabel;
private String statement;
private String sampleData;
private Map<String, Object> results;
+ public String getPatternLabel() {
+ return patternLabel;
+ }
+
+ public void setPatternLabel(String patternLabel) {
+ this.patternLabel = patternLabel;
+ }
+
public String getStatement() {
return statement;
}
@@ -60,6 +69,7 @@ public class GrokValidation {
GrokValidation that = (GrokValidation) o;
+ if (patternLabel != null ? !patternLabel.equals(that.patternLabel) : that.patternLabel != null) return false;
if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false;
if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false;
return results != null ? results.equals(that.results) : that.results == null;
@@ -67,7 +77,8 @@ public class GrokValidation {
@Override
public int hashCode() {
- int result = statement != null ? statement.hashCode() : 0;
+ int result = patternLabel != null ? patternLabel.hashCode() : 0;
+ result = 31 * result + (statement != null ? statement.hashCode() : 0);
result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0);
result = 31 * result + (results != null ? results.hashCode() : 0);
return result;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
index 8dfe17b..50eb88d 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
@@ -22,6 +22,7 @@ import org.apache.metron.common.configuration.SensorParserConfig;
public class ParseMessageRequest {
private SensorParserConfig sensorParserConfig;
+ private String grokStatement;
private String sampleData;
public SensorParserConfig getSensorParserConfig() {
@@ -32,6 +33,14 @@ public class ParseMessageRequest {
this.sensorParserConfig = sensorParserConfig;
}
+ public String getGrokStatement() {
+ return grokStatement;
+ }
+
+ public void setGrokStatement(String grokStatement) {
+ this.grokStatement = grokStatement;
+ }
+
public String getSampleData() {
return sampleData;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
new file mode 100644
index 0000000..cb10cfe
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SensorParserContext {
+
+ private Map<String, Object> sampleData;
+ private SensorParserConfig sensorParserConfig;
+
+ public Map<String, Object> getSampleData() {
+ if (sampleData == null) {
+ return new HashMap<>();
+ }
+ return sampleData;
+ }
+
+ public void setSampleData(Map<String, Object> sampleData) {
+ this.sampleData = sampleData;
+ }
+
+ public SensorParserConfig getSensorParserConfig() {
+ if (sensorParserConfig == null) {
+ return new SensorParserConfig();
+ }
+ return sensorParserConfig;
+ }
+
+ public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
+ this.sensorParserConfig = sensorParserConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
index 6894e89..0ad2e9f 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
@@ -39,4 +39,22 @@ public class TopologyResponse {
this.status = TopologyResponseCode.ERROR;
this.message = message;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologyResponse that = (TopologyResponse) o;
+
+ if (status != null ? !status.equals(that.status) : that.status != null) return false;
+ return message != null ? message.equals(that.message) : that.message == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = status != null ? status.hashCode() : 0;
+ result = 31 * result + (message != null ? message.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
index 0e2d28a..6fc1c01 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
@@ -26,8 +26,8 @@ public class TopologyStatus {
private String name;
private TopologyStatusCode status;
private Map<String, Object>[] topologyStats;
- private double latency = 0;
- private double throughput = 0;
+ private Double latency = 0.0;
+ private Double throughput = 0.0;
public String getId() {
return id;
@@ -73,4 +73,28 @@ public class TopologyStatus {
}
}
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologyStatus that = (TopologyStatus) o;
+
+ if (id != null ? !id.equals(that.id) : that.id != null) return false;
+ if (name != null ? !name.equals(that.name) : that.name != null) return false;
+ if (status != null ? !status.equals(that.status) : that.status != null) return false;
+ if (!latency.equals(that.latency)) return false;
+ return throughput.equals(that.throughput);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id != null ? id.hashCode() : 0;
+ result = 31 * result + (name != null ? name.hashCode() : 0);
+ result = 31 * result + (status != null ? status.hashCode() : 0);
+ result = 31 * result + (latency != null ? latency.hashCode() : 0);
+ result = 31 * result + (throughput != null ? throughput.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
index b9d39a4..8621daf 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.rest.model;
+import java.util.Arrays;
+
public class TopologySummary {
private TopologyStatus[] topologies;
@@ -31,4 +33,19 @@ public class TopologySummary {
public void setTopologies(TopologyStatus[] topologies) {
this.topologies = topologies;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologySummary that = (TopologySummary) o;
+
+ return topologies != null ? Arrays.equals(topologies, that.topologies) : that.topologies != null;
+ }
+
+ @Override
+ public int hashCode() {
+ return topologies != null ? Arrays.hashCode(topologies) : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
deleted file mode 100644
index c2e39e4..0000000
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.model;
-
-import org.apache.metron.common.configuration.SensorParserConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class TransformationValidation {
-
- private Map<String, Object> sampleData;
- private SensorParserConfig sensorParserConfig;
-
- public Map<String, Object> getSampleData() {
- if (sampleData == null) {
- return new HashMap<>();
- }
- return sampleData;
- }
-
- public void setSampleData(Map<String, Object> sampleData) {
- this.sampleData = sampleData;
- }
-
- public SensorParserConfig getSensorParserConfig() {
- if (sensorParserConfig == null) {
- return new SensorParserConfig();
- }
- return sensorParserConfig;
- }
-
- public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
- this.sensorParserConfig = sensorParserConfig;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/.gitignore
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/.gitignore b/metron-interface/metron-rest/.gitignore
new file mode 100644
index 0000000..26b3af1
--- /dev/null
+++ b/metron-interface/metron-rest/.gitignore
@@ -0,0 +1 @@
+metrondb.*.db
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 63a2d34..cf41bf3 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -1,66 +1,83 @@
-# Metron REST and Configuration UI
+# Metron REST
-This UI exposes and aids in sensor configuration.
+This module provides a RESTful API for interacting with Metron.
## Prerequisites
* A running Metron cluster
-* A running instance of MySQL
* Java 8 installed
* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
## Installation
-1. Package the Application with Maven:
- ```
- mvn clean package
- ```
+1. Package the application with Maven:
+```
+mvn clean package
+```
1. Untar the archive in the target directory. The directory structure will look like:
- ```
- bin
- start.sh
- lib
- metron-rest-version.jar
- ```
+```
+bin
+ start_metron_rest.sh
+lib
+ metron-rest-$METRON_VERSION.jar
+```
+
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
-1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
- ```
- export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
- ```
+1. Start the application with this command:
+```
+./bin/start_metron_rest.sh /path/to/application.yml
+```
-1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
- ```
- export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
- ```
+## Usage
-1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
-1. Create a Config UI database in MySQL with this command:
- ```
- CREATE DATABASE IF NOT EXISTS metronrest
- ```
+## Security
-1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users:
+```
+./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+```
-1. Start the UI with this command:
- ```
- ./bin/start.sh /path/to/application.yml
- ```
+For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows:
-## Usage
+1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement:
+1. Connect to MySQL and create a Metron REST database:
+```
+CREATE DATABASE IF NOT EXISTS metronrest
+```
+
+1. Add users:
```
use metronrest;
insert into users (username, password, enabled) values ('your_username','your_password',1);
insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
```
-Users can be added to additional groups with this SQL statement:
+
+1. Replace the H2 connection information in the application.yml file with MySQL connection information:
```
-use metronrest;
-insert into authorities (username, authority) values ('your_username', 'your_group');
+spring:
+ datasource:
+ driverClassName: com.mysql.jdbc.Driver
+ url: jdbc:mysql://mysql_host:3306/metronrest
+ username: metron_rest_user
+ password: metron_rest_password
+ platform: mysql
```
+1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
+```
+<dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.client.version}</version>
+</dependency>
+```
+
+1. Follow the steps in the [Installation](#installation) section
+
## API
Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI.
@@ -72,6 +89,10 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `POST /api/v1/global/config`](#post-apiv1globalconfig)|
| [ `GET /api/v1/grok/list`](#get-apiv1groklist)|
| [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)|
+| [ `POST /api/v1/hdfs`](#post-apiv1hdfs)|
+| [ `GET /api/v1/hdfs`](#get-apiv1hdfs)|
+| [ `DELETE /api/v1/hdfs`](#delete-apiv1hdfs)|
+| [ `GET /api/v1/hdfs/list`](#get-apiv1hdfslist)|
| [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)|
| [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)|
| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})|
@@ -93,6 +114,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)|
| [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})|
| [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})|
+| [ `POST /api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)|
+| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)|
+| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)|
+| [ `GET /api/v1/stellar/list/simple/functions`](#get-apiv1stellarlistsimplefunctions)|
+| [ `POST /api/v1/stellar/validate/rules`](#post-apiv1stellarvalidaterules)|
| [ `GET /api/v1/storm`](#get-apiv1storm)|
| [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)|
| [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)|
@@ -110,11 +136,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})|
| [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})|
| [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})|
-| [ `GET /api/v1/transformation/list`](#get-apiv1transformationlist)|
-| [ `GET /api/v1/transformation/list/functions`](#get-apiv1transformationlistfunctions)|
-| [ `GET /api/v1/transformation/list/simple/functions`](#get-apiv1transformationlistsimplefunctions)|
-| [ `POST /api/v1/transformation/validate`](#post-apiv1transformationvalidate)|
-| [ `POST /api/v1/transformation/validate/rules`](#post-apiv1transformationvalidaterules)|
| [ `GET /api/v1/user`](#get-apiv1user)|
### `GET /api/v1/global/config`
@@ -145,10 +166,41 @@ Request and Response objects are JSON formatted. The JSON schemas are available
### `POST /api/v1/grok/validate`
* Description: Applies a Grok statement to a sample message
* Input:
- * grokValidation - Object containing Grok statment and sample message
+ * grokValidation - Object containing Grok statement and sample message
* Returns:
* 200 - JSON results
+### `POST /api/v1/hdfs`
+ * Description: Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.
+ * Input:
+ * path - Path to HDFS file
+ * contents - File contents
+ * Returns:
+ * 200 - Contents were written
+
+### `GET /api/v1/hdfs`
+ * Description: Reads a file from HDFS and returns the contents
+ * Input:
+ * path - Path to HDFS file
+ * Returns:
+ * 200 - Returns file contents
+
+### `DELETE /api/v1/hdfs`
+ * Description: Deletes a file from HDFS
+ * Input:
+ * path - Path to HDFS file
+ * recursive - Delete files recursively
+ * Returns:
+ * 200 - File was deleted
+ * 404 - File was not found in HDFS
+
+### `GET /api/v1/hdfs/list`
+ * Description: Reads a file from HDFS and returns the contents
+ * Input:
+ * path - Path to HDFS directory
+ * Returns:
+ * 200 - Returns file contents
+
### `GET /api/v1/kafka/topic`
* Description: Retrieves all Kafka topics
* Returns:
@@ -296,6 +348,35 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns SensorParserConfig
* 404 - SensorParserConfig is missing
+### `POST /api/v1/stellar/apply/transformations`
+ * Description: Executes transformations against a sample message
+ * Input:
+ * transformationValidation - Object containing SensorParserConfig and sample message
+ * Returns:
+ * 200 - Returns transformation results
+
+### `GET /api/v1/stellar/list`
+ * Description: Retrieves field transformations
+ * Returns:
+ * 200 - Returns a list field transformations
+
+### `GET /api/v1/stellar/list/functions`
+ * Description: Lists the Stellar functions that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of Stellar functions
+
+### `GET /api/v1/stellar/list/simple/functions`
+ * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of simple Stellar functions
+
+### `POST /api/v1/stellar/validate/rules`
+ * Description: Tests Stellar statements to ensure they are well-formed
+ * Input:
+ * statements - List of statements to validate
+ * Returns:
+ * 200 - Returns validation results
+
### `GET /api/v1/storm`
* Description: Retrieves the status of all Storm topologies
* Returns:
@@ -399,40 +480,43 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns topology status information
* 404 - Topology is missing
-### `GET /api/v1/transformation/list`
- * Description: Retrieves field transformations
+### `GET /api/v1/user`
+ * Description: Retrieves the current user
* Returns:
- * 200 - Returns a list field transformations
+ * 200 - Current user
-### `GET /api/v1/transformation/list/functions`
- * Description: Lists the Stellar functions that can be found on the classpath
- * Returns:
- * 200 - Returns a list of Stellar functions
+## Testing
-### `GET /api/v1/transformation/list/simple/functions`
- * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
- * Returns:
- * 200 - Returns a list of simple Stellar functions
+Profiles are includes for both the metron-docker and Quick Dev environments.
-### `POST /api/v1/transformation/validate`
- * Description: Executes transformations against a sample message
- * Input:
- * transformationValidation - Object containing SensorParserConfig and sample message
- * Returns:
- * 200 - Returns transformation results
+### metron-docker
-### `POST /api/v1/transformation/validate/rules`
- * Description: Tests Stellar statements to ensure they are well-formed
- * Input:
- * statements - List of statements to validate
- * Returns:
- * 200 - Returns validation results
+Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=docker,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
-### `GET /api/v1/user`
- * Description: Retrieves the current user
- * Returns:
- * 200 - Current user
+### Quick Dev
+Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=vagrant,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
+
+To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
+```
+mvn clean package
+scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/
+```
+Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari:
+```
+java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+```
+The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.
## License
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index c2beb2d..849455e 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.3.0</version>
+ <version>0.3.1</version>
</parent>
<artifactId>metron-rest</artifactId>
<properties>
@@ -61,15 +61,15 @@
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.client.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -105,6 +105,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -122,8 +126,16 @@
</exclusion>
<exclusion>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-writer</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -147,18 +159,6 @@
<version>0.1.0</version>
</dependency>
<dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>0.9.10</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
@@ -166,7 +166,6 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
new file mode 100644
index 0000000..bf7ae84
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.HdfsService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+@RestController
+@RequestMapping("/api/v1/hdfs")
+public class HdfsController {
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+ @ApiResponse(message = "Returns file contents", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<List<String>> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException {
+ return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+ @ApiResponse(message = "Returns file contents", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<String> read(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path) throws RestException {
+ String contents = hdfsService.read(new Path(path));
+ if (contents != null) {
+ return new ResponseEntity<>(hdfsService.read(new Path(path)), HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+
+ }
+
+ @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.")
+ @ApiResponse(message = "Contents were written", code = 200)
+ @RequestMapping(method = RequestMethod.POST)
+ ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path,
+ @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException {
+ hdfsService.write(new Path(path), contents.getBytes(UTF_8));
+ return new ResponseEntity<>(HttpStatus.OK);
+
+ }
+
+ @ApiOperation(value = "Deletes a file from HDFS")
+ @ApiResponses(value = { @ApiResponse(message = "File was deleted", code = 200),
+ @ApiResponse(message = "File was not found in HDFS", code = 404) })
+ @RequestMapping(method = RequestMethod.DELETE)
+ ResponseEntity<Boolean> delete(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path,
+ @ApiParam(name = "recursive", value = "Delete files recursively") @RequestParam(required = false, defaultValue = "false") boolean recursive) throws RestException {
+ if (hdfsService.delete(new Path(path), recursive)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
new file mode 100644
index 0000000..13327f9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/stellar")
+public class StellarController {
+
+ @Autowired
+ private StellarService stellarService;
+
+ @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
+ @ApiResponse(message = "Returns validation results", code = 200)
+ @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Boolean>> validateRules(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
+ return new ResponseEntity<>(stellarService.validateRules(statements), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Executes transformations against a sample message")
+ @ApiResponse(message = "Returns transformation results", code = 200)
+ @RequestMapping(value = "/apply/transformations", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Object>> applyTransformations(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody SensorParserContext sensorParserContext) throws RestException {
+ return new ResponseEntity<>(stellarService.applyTransformations(sensorParserContext), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves field transformations")
+ @ApiResponse(message = "Returns a list field transformations", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<FieldTransformations[]> list() throws RestException {
+ return new ResponseEntity<>(stellarService.getTransformations(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
+ @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
+ return new ResponseEntity<>(stellarService.getStellarFunctions(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
+ @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
+ return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
deleted file mode 100644
index bc8b201..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.controller;
-
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import io.swagger.annotations.ApiResponse;
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.RestException;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-import org.apache.metron.rest.service.TransformationService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.List;
-import java.util.Map;
-
-@RestController
-@RequestMapping("/api/v1/transformation")
-public class TransformationController {
-
- @Autowired
- private TransformationService transformationService;
-
- @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
- @ApiResponse(message = "Returns validation results", code = 200)
- @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
- ResponseEntity<Map<String, Boolean>> validateRule(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
- return new ResponseEntity<>(transformationService.validateRules(statements), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Executes transformations against a sample message")
- @ApiResponse(message = "Returns transformation results", code = 200)
- @RequestMapping(value = "/validate", method = RequestMethod.POST)
- ResponseEntity<Map<String, Object>> validateTransformation(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody TransformationValidation transformationValidation) throws RestException {
- return new ResponseEntity<>(transformationService.validateTransformation(transformationValidation), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Retrieves field transformations")
- @ApiResponse(message = "Returns a list field transformations", code = 200)
- @RequestMapping(value = "/list", method = RequestMethod.GET)
- ResponseEntity<FieldTransformations[]> list() throws RestException {
- return new ResponseEntity<>(transformationService.getTransformations(), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
- @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
- @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
- ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
- return new ResponseEntity<>(transformationService.getStellarFunctions(), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
- @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
- @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
- ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
- return new ResponseEntity<>(transformationService.getSimpleStellarFunctions(), HttpStatus.OK);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index 95ce1ba..268d396 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.service;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
+import java.io.File;
import java.util.Map;
public interface GrokService {
@@ -28,4 +29,6 @@ public interface GrokService {
GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
+ File saveTemporary(String statement, String name) throws RestException;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index 97888ff..d5932c7 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -17,19 +17,18 @@
*/
package org.apache.metron.rest.service;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.springframework.stereotype.Service;
+import org.apache.metron.rest.RestException;
-import java.io.IOException;
+import java.util.List;
public interface HdfsService {
- byte[] read(Path path) throws IOException;
+ String read(Path path) throws RestException;
- void write(Path path, byte[] contents) throws IOException;
+ void write(Path path, byte[] contents) throws RestException;
- FileStatus[] list(Path path) throws IOException;
+ List<String> list(Path path) throws RestException;
- boolean delete(Path path, boolean recursive) throws IOException;
+ boolean delete(Path path, boolean recursive) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
index efda639..9b863b8 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
@@ -22,6 +22,7 @@ import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.json.simple.JSONObject;
+import java.util.List;
import java.util.Map;
public interface SensorParserConfigService {
@@ -32,6 +33,8 @@ public interface SensorParserConfigService {
Iterable<SensorParserConfig> getAll() throws RestException;
+ List<String> getAllTypes() throws RestException;
+
boolean delete(String name) throws RestException;
Map<String, String> getAvailableParsers();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
new file mode 100644
index 0000000..14cd31f
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+
+import java.util.List;
+import java.util.Map;
+
+public interface StellarService {
+
+ Map<String, Boolean> validateRules(List<String> rules);
+
+ Map<String, Object> applyTransformations(SensorParserContext sensorParserContext);
+
+ FieldTransformations[] getTransformations();
+
+ List<StellarFunctionDescription> getStellarFunctions();
+
+ List<StellarFunctionDescription> getSimpleStellarFunctions();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
deleted file mode 100644
index d1400c6..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-
-import java.util.List;
-import java.util.Map;
-
-public interface TransformationService {
-
- Map<String, Boolean> validateRules(List<String> rules);
-
- Map<String, Object> validateTransformation(TransformationValidation transformationValidation);
-
- FieldTransformations[] getTransformations();
-
- List<StellarFunctionDescription> getStellarFunctions();
-
- List<StellarFunctionDescription> getSimpleStellarFunctions();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 323ca78..3f2de2f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -20,22 +20,35 @@ package org.apache.metron.rest.service.impl;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.directory.api.util.Strings;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Map;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+
@Service
public class GrokServiceImpl implements GrokService {
+
+ private Environment environment;
+
private Grok commonGrok;
@Autowired
- public GrokServiceImpl(Grok commonGrok) {
+ public GrokServiceImpl(Environment environment, Grok commonGrok) {
+ this.environment = environment;
this.commonGrok = commonGrok;
}
@@ -48,20 +61,21 @@ public class GrokServiceImpl implements GrokService {
public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException {
Map<String, Object> results;
try {
- String statement = Strings.isEmpty(grokValidation.getStatement()) ? "" : grokValidation.getStatement();
-
+ if (grokValidation.getPatternLabel() == null) {
+ throw new RestException("Pattern label is required");
+ }
+ if (Strings.isEmpty(grokValidation.getStatement())) {
+ throw new RestException("Grok statement is required");
+ }
Grok grok = new Grok();
grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
- grok.addPatternFromReader(new StringReader(statement));
- String patternLabel = statement.substring(0, statement.indexOf(" "));
- String grokPattern = "%{" + patternLabel + "}";
+ grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
+ String grokPattern = "%{" + grokValidation.getPatternLabel() + "}";
grok.compile(grokPattern);
Match gm = grok.match(grokValidation.getSampleData());
gm.captures();
results = gm.toMap();
- results.remove(patternLabel);
- } catch (StringIndexOutOfBoundsException e) {
- throw new RestException("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)", e.getCause());
+ results.remove(grokValidation.getPatternLabel());
} catch (Exception e) {
throw new RestException(e);
}
@@ -69,4 +83,31 @@ public class GrokServiceImpl implements GrokService {
return grokValidation;
}
+ @Override
+ public File saveTemporary(String statement, String name) throws RestException {
+ if (statement != null) {
+ try {
+ File grokDirectory = new File(getTemporaryGrokRootPath());
+ if (!grokDirectory.exists()) {
+ grokDirectory.mkdirs();
+ }
+ File path = new File(grokDirectory, name);
+ FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
+ fileWriter.write(statement);
+ fileWriter.close();
+ return path;
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ } else {
+ throw new RestException("A grokStatement must be provided");
+ }
+ }
+
+ private String getTemporaryGrokRootPath() {
+ String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
+ Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+ return new Path(grokTempPath, authentication.getName()).toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index c14ec0c..789c421 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -19,44 +19,73 @@ package org.apache.metron.rest.service.impl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.HdfsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
@Service
public class HdfsServiceImpl implements HdfsService {
- @Autowired
private Configuration configuration;
+ @Autowired
+ public HdfsServiceImpl(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
@Override
- public byte[] read(Path path) throws IOException {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
- return byteArrayOutputStream.toByteArray();
+ public List<String> list(Path path) throws RestException {
+ try {
+ return Arrays.asList(FileSystem.get(configuration).listStatus(path)).stream().map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
@Override
- public void write(Path path, byte[] contents) throws IOException {
- FSDataOutputStream fsDataOutputStream = FileSystem.get(configuration).create(path, true);
- fsDataOutputStream.write(contents);
- fsDataOutputStream.close();
+ public String read(Path path) throws RestException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ try {
+ IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
+ } catch (FileNotFoundException e) {
+ return null;
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ return new String(byteArrayOutputStream.toByteArray(), UTF_8);
}
@Override
- public FileStatus[] list(Path path) throws IOException {
- return FileSystem.get(configuration).listStatus(path);
+ public void write(Path path, byte[] contents) throws RestException {
+ FSDataOutputStream fsDataOutputStream;
+ try {
+ fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+ fsDataOutputStream.write(contents);
+ fsDataOutputStream.close();
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
@Override
- public boolean delete(Path path, boolean recursive) throws IOException {
+ public boolean delete(Path path, boolean recursive) throws RestException {
+ try {
return FileSystem.get(configuration).delete(path, recursive);
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
index 8b3dbb7..2bfef89 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -36,12 +36,16 @@ import java.util.Map;
@Service
public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
- @Autowired
private ObjectMapper objectMapper;
- @Autowired
private CuratorFramework client;
+ @Autowired
+ public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+ this.objectMapper = objectMapper;
+ this.client = client;
+ }
+
@Override
public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index ab46418..9f984e0 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -38,12 +38,16 @@ import java.util.Map;
@Service
public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
- @Autowired
private ObjectMapper objectMapper;
- @Autowired
private CuratorFramework client;
+ @Autowired
+ public SensorIndexingConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+ this.objectMapper = objectMapper;
+ this.client = client;
+ }
+
@Override
public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index cb88708..eddfc8d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,10 +17,8 @@
*/
package org.apache.metron.rest.service.impl;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
@@ -28,20 +26,15 @@ import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
-import org.apache.metron.rest.service.HdfsService;
+import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
import org.reflections.Reflections;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.env.Environment;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,72 +42,40 @@ import java.util.Map;
import java.util.Set;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-import static org.apache.metron.rest.MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATH_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATTERN_LABEL_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_STATEMENT_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
@Service
public class SensorParserConfigServiceImpl implements SensorParserConfigService {
- @Autowired
- private Environment environment;
-
- @Autowired
private ObjectMapper objectMapper;
private CuratorFramework client;
+ private GrokService grokService;
+
@Autowired
- public void setClient(CuratorFramework client) {
+ public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) {
+ this.objectMapper = objectMapper;
this.client = client;
+ this.grokService = grokService;
}
- @Autowired
- private HdfsService hdfsService;
-
private Map<String, String> availableParsers;
@Override
public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
- String serializedConfig;
- if (isGrokConfig(sensorParserConfig)) {
- addGrokPathToConfig(sensorParserConfig);
- sensorParserConfig.getParserConfig().putIfAbsent(MetronRestConstants.GROK_PATTERN_LABEL_KEY, sensorParserConfig.getSensorTopic().toUpperCase());
- String statement = (String) sensorParserConfig.getParserConfig().remove(MetronRestConstants.GROK_STATEMENT_KEY);
- serializedConfig = serialize(sensorParserConfig);
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_STATEMENT_KEY, statement);
- saveGrokStatement(sensorParserConfig);
- } else {
- serializedConfig = serialize(sensorParserConfig);
- }
try {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), serializedConfig.getBytes(), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
} catch (Exception e) {
throw new RestException(e);
}
return sensorParserConfig;
}
- private String serialize(SensorParserConfig sensorParserConfig) throws RestException {
- String serializedConfig;
- try {
- serializedConfig = objectMapper.writeValueAsString(sensorParserConfig);
- } catch (JsonProcessingException e) {
- throw new RestException("Could not serialize SensorParserConfig", "Could not serialize " + sensorParserConfig.toString(), e.getCause());
- }
- return serializedConfig;
- }
-
@Override
public SensorParserConfig findOne(String name) throws RestException {
SensorParserConfig sensorParserConfig;
try {
sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client);
- if (isGrokConfig(sensorParserConfig)) {
- addGrokStatementToConfig(sensorParserConfig);
- }
} catch (KeeperException.NoNodeException e) {
return null;
} catch (Exception e) {
@@ -145,7 +106,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
return true;
}
- private List<String> getAllTypes() throws RestException {
+ @Override
+ public List<String> getAllTypes() throws RestException {
List<String> types;
try {
types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot());
@@ -190,20 +152,21 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a parserClassName");
} else {
- MessageParser<JSONObject> parser = null;
+ MessageParser<JSONObject> parser;
try {
parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
+ File temporaryGrokFile = null;
if (isGrokConfig(sensorParserConfig)) {
- saveTemporaryGrokStatement(sensorParserConfig);
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()).toString());
+ temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic());
+ sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString());
}
parser.configure(sensorParserConfig.getParserConfig());
JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
- if (isGrokConfig(sensorParserConfig)) {
- deleteTemporaryGrokStatement(sensorParserConfig);
+ if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) {
+ temporaryGrokFile.delete();
}
return results;
}
@@ -212,78 +175,4 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
private boolean isGrokConfig(SensorParserConfig sensorParserConfig) {
return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName());
}
-
- private void addGrokStatementToConfig(SensorParserConfig sensorParserConfig) throws RestException {
- String grokStatement = "";
- String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
- if (grokPath != null) {
- String fullGrokStatement = getGrokStatement(grokPath);
- String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
- grokStatement = fullGrokStatement.replaceFirst(patternLabel + " ", "");
- }
- sensorParserConfig.getParserConfig().put(GROK_STATEMENT_KEY, grokStatement);
- }
-
- private void addGrokPathToConfig(SensorParserConfig sensorParserConfig) {
- if (sensorParserConfig.getParserConfig().get(GROK_PATH_KEY) == null) {
- String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
- if (grokStatement != null) {
- sensorParserConfig.getParserConfig().put(GROK_PATH_KEY,
- new Path(environment.getProperty(GROK_DEFAULT_PATH_SPRING_PROPERTY), sensorParserConfig.getSensorTopic()).toString());
- }
- }
- }
-
- private String getGrokStatement(String path) throws RestException {
- try {
- return new String(hdfsService.read(new Path(path)));
- } catch (IOException e) {
- throw new RestException(e);
- }
- }
-
- private void saveGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
- saveGrokStatement(sensorParserConfig, false);
- }
-
- private void saveTemporaryGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
- saveGrokStatement(sensorParserConfig, true);
- }
-
- private void saveGrokStatement(SensorParserConfig sensorParserConfig, boolean isTemporary) throws RestException {
- String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
- String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
- String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
- if (grokStatement != null) {
- String fullGrokStatement = patternLabel + " " + grokStatement;
- try {
- if (!isTemporary) {
- hdfsService.write(new Path(grokPath), fullGrokStatement.getBytes());
- } else {
- File grokDirectory = new File(getTemporaryGrokRootPath());
- if (!grokDirectory.exists()) {
- grokDirectory.mkdirs();
- }
- FileWriter fileWriter = new FileWriter(new File(grokDirectory, sensorParserConfig.getSensorTopic()));
- fileWriter.write(fullGrokStatement);
- fileWriter.close();
- }
- } catch (IOException e) {
- throw new RestException(e);
- }
- } else {
- throw new RestException("A grokStatement must be provided");
- }
- }
-
- private void deleteTemporaryGrokStatement(SensorParserConfig sensorParserConfig) {
- File file = new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic());
- file.delete();
- }
-
- private String getTemporaryGrokRootPath() {
- String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
- Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
- return new Path(grokTempPath, authentication.getName()).toString();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
new file mode 100644
index 0000000..f5392be
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctionInfo;
+import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.json.simple.JSONObject;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+public class StellarServiceImpl implements StellarService {
+
+ @Override
+ public Map<String, Boolean> validateRules(List<String> rules) {
+ Map<String, Boolean> results = new HashMap<>();
+ StellarProcessor stellarProcessor = new StellarProcessor();
+ for(String rule: rules) {
+ try {
+ boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
+ results.put(rule, result);
+ } catch (ParseException e) {
+ results.put(rule, false);
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) {
+ JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData());
+ sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
+ fieldTransformer.transformAndUpdate(sampleJson, sensorParserContext.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
+ }
+ );
+ return sampleJson;
+ }
+
+ @Override
+ public FieldTransformations[] getTransformations() {
+ return FieldTransformations.values();
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
+ Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
+ stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
+ stellarFunctionDescriptions.add(new StellarFunctionDescription(
+ stellarFunctionInfo.getName(),
+ stellarFunctionInfo.getDescription(),
+ stellarFunctionInfo.getParams(),
+ stellarFunctionInfo.getReturns()));
+ });
+ return stellarFunctionDescriptions;
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getSimpleStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
+ return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
+ stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
index cb7c449..9bd368f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -33,16 +33,17 @@ public class StormAdminServiceImpl implements StormAdminService {
private StormCLIWrapper stormCLIClientWrapper;
+ private GlobalConfigService globalConfigService;
+
+ private SensorParserConfigService sensorParserConfigService;
+
@Autowired
- public void setStormCLIClientWrapper(StormCLIWrapper stormCLIClientWrapper) {
+ public StormAdminServiceImpl(StormCLIWrapper stormCLIClientWrapper, GlobalConfigService globalConfigService, SensorParserConfigService sensorParserConfigService) {
this.stormCLIClientWrapper = stormCLIClientWrapper;
+ this.globalConfigService = globalConfigService;
+ this.sensorParserConfigService = sensorParserConfigService;
}
- @Autowired
- private GlobalConfigService globalConfigService;
-
- @Autowired
- private SensorParserConfigService sensorParserConfigService;
@Override
public TopologyResponse startParserTopology(String name) throws RestException {
[02/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
new file mode 100644
index 0000000..0299708
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.integration.components.KafkaComponent;
+import org.apache.metron.rest.generator.SampleDataGenerator;
+import org.apache.metron.rest.service.KafkaService;
+import org.hamcrest.Matchers;
+import org.json.simple.parser.ParseException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import java.io.IOException;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class KafkaControllerIntegrationTest {
+
+ @Autowired
+ private KafkaComponent kafkaWithZKComponent;
+
+ class SampleDataRunner implements Runnable {
+
+ private boolean stop = false;
+ private String path = "../../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput";
+
+ @Override
+ public void run() {
+ SampleDataGenerator broSampleDataGenerator = new SampleDataGenerator();
+ broSampleDataGenerator.setBrokerUrl(kafkaWithZKComponent.getBrokerList());
+ broSampleDataGenerator.setNum(1);
+ broSampleDataGenerator.setSelectedSensorType("bro");
+ broSampleDataGenerator.setDelay(0);
+ try {
+ while(!stop) {
+ broSampleDataGenerator.generateSampleData(path);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void stop() {
+ stop = true;
+ }
+ }
+
+ private SampleDataRunner sampleDataRunner = new SampleDataRunner();
+ private Thread sampleDataThread = new Thread(sampleDataRunner);
+
+ /**
+ {
+ "name": "bro",
+ "numPartitions": 1,
+ "properties": {},
+ "replicationFactor": 1
+ }
+ */
+ @Multiline
+ public static String broTopic;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ @Autowired
+ private KafkaService kafkaService;
+
+ private MockMvc mockMvc;
+
+ private String kafkaUrl = "/api/v1/kafka";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(kafkaUrl + "/topic").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broTopic))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.kafkaService.deleteTopic("bro");
+ this.kafkaService.deleteTopic("someTopic");
+ Thread.sleep(1000);
+
+ this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(post(kafkaUrl + "/topic").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broTopic))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value("bro"))
+ .andExpect(jsonPath("$.numPartitions").value(1))
+ .andExpect(jsonPath("$.replicationFactor").value(1));
+
+ sampleDataThread.start();
+ Thread.sleep(1000);
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value("bro"))
+ .andExpect(jsonPath("$.numPartitions").value(1))
+ .andExpect(jsonPath("$.replicationFactor").value(1));
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", Matchers.hasItem("bro")));
+
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
+ .andExpect(jsonPath("$").isNotEmpty());
+
+ this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic/sample").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+ }
+
+ @After
+ public void tearDown() {
+ sampleDataRunner.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
new file mode 100644
index 0000000..a659772
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.rest.service.SensorEnrichmentConfigService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class SensorEnrichmentConfigControllerIntegrationTest {
+
+ /**
+ {
+ "enrichment": {
+ "fieldMap": {
+ "geo": [
+ "ip_dst_addr"
+ ],
+ "host": [
+ "ip_dst_addr"
+ ],
+ "hbaseEnrichment": [
+ "ip_src_addr"
+ ],
+ "stellar": {
+ "config": {
+ "group1": {
+ "foo": "1 + 1",
+ "bar": "foo"
+ },
+ "group2": {
+ "ALL_CAPS": "TO_UPPER(source.type)"
+ }
+ }
+ }
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr": [
+ "sample"
+ ]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr": [
+ "malicious_ip"
+ ],
+ "ip_dst_addr": [
+ "malicious_ip"
+ ]
+ },
+ "triageConfig": {
+ "riskLevelRules": {
+ "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'": 10
+ },
+ "aggregator": "MAX"
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ @Autowired
+ private SensorEnrichmentConfigService sensorEnrichmentConfigService;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String sensorEnrichmentConfigUrl = "/api/v1/sensor/enrichment/config";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(sensorEnrichmentConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ sensorEnrichmentConfigService.delete("broTest");
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().bytes("{}".getBytes()));
+
+ this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
+
+ this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo"))
+ .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip"))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"]").value(10))
+ .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX"));
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.broTest.enrichment.fieldMap.geo[0] == 'ip_dst_addr' &&" +
+ "@.broTest.enrichment.fieldMap.host[0] == 'ip_dst_addr' &&" +
+ "@.broTest.enrichment.fieldMap.hbaseEnrichment[0] == 'ip_src_addr' &&" +
+ "@.broTest.enrichment.fieldToTypeMap.ip_src_addr[0] == 'sample' &&" +
+ "@.broTest.enrichment.fieldMap.stellar.config.group1.foo == '1 + 1' &&" +
+ "@.broTest.enrichment.fieldMap.stellar.config.group1.bar == 'foo' &&" +
+ "@.broTest.enrichment.fieldMap.stellar.config.group2.ALL_CAPS == 'TO_UPPER(source.type)' &&" +
+ "@.broTest.threatIntel.fieldMap.hbaseThreatIntel[0] == 'ip_src_addr' &&" +
+ "@.broTest.threatIntel.fieldMap.hbaseThreatIntel[1] == 'ip_dst_addr' &&" +
+ "@.broTest.threatIntel.fieldToTypeMap.ip_src_addr[0] == 'malicious_ip' &&" +
+ "@.broTest.threatIntel.fieldToTypeMap.ip_dst_addr[0] == 'malicious_ip' &&" +
+ "@.broTest.threatIntel.triageConfig.riskLevelRules[\"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\"] == 10 &&" +
+ "@.broTest.threatIntel.triageConfig.aggregator == 'MAX'" +
+ ")]").exists());
+
+ this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+
+ this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/list/available").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[0]").value("geo"))
+ .andExpect(jsonPath("$[1]").value("host"))
+ .andExpect(jsonPath("$[2]").value("whois"));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
new file mode 100644
index 0000000..10edadc
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.rest.service.SensorIndexingConfigService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class SensorIndexingConfigControllerIntegrationTest {
+
+ /**
+ {
+ "index": "broTest",
+ "batchSize": 1
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ @Autowired
+ private SensorIndexingConfigService sensorIndexingConfigService;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String sensorIndexingConfigUrl = "/api/v1/sensor/indexing/config";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(sensorIndexingConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ sensorIndexingConfigService.delete("broTest");
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().bytes("{}".getBytes()));
+
+ this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.index").value("broTest"))
+ .andExpect(jsonPath("$.batchSize").value(1));
+
+ this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.index").value("broTest"))
+ .andExpect(jsonPath("$.batchSize").value(1));
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.index").value("broTest"))
+ .andExpect(jsonPath("$.batchSize").value(1));
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.broTest.index == 'broTest' &&" +
+ "@.broTest.batchSize == 1" +
+ ")]").exists());
+
+ this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
new file mode 100644
index 0000000..09b1b0a
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.io.FileUtils;
+import org.apache.metron.rest.MetronRestConstants;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.env.Environment;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class SensorParserConfigControllerIntegrationTest {
+
+ /**
+ {
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squidTest",
+ "parserConfig": {
+ "patternLabel": "SQUIDTEST",
+ "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+ "timestampField": "timestamp"
+ },
+ "fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "full_hostname", "domain_without_subdomains" ]
+ ,"config" : {
+ "full_hostname" : "URL_TO_HOST(url)"
+ ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String squidJson;
+
+ /**
+ {
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squidTest",
+ "parserConfig": {
+ "patternLabel": "SQUIDTEST",
+ "timestampField": "timestamp"
+ }
+ }
+ */
+ @Multiline
+ public static String missingGrokJson;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic":"broTest",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ /**
+ {
+ "sensorParserConfig":
+ {
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squidTest",
+ "parserConfig": {
+ "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+ "patternLabel": "SQUIDTEST",
+ "grokPath":"./squidTest",
+ "timestampField": "timestamp"
+ }
+ },
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
+ }
+ */
+ @Multiline
+ public static String parseRequest;
+
+ /**
+ {
+ "sensorParserConfig": null,
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
+ }
+ */
+ @Multiline
+ public static String missingConfigParseRequest;
+
+ /**
+ {
+ "sensorParserConfig":
+ {
+ "sensorTopic": "squidTest",
+ "parserConfig": {
+ "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+ "patternLabel": "SQUIDTEST",
+ "grokPath":"./squidTest",
+ "timestampField": "timestamp"
+ }
+ },
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
+ }
+ */
+ @Multiline
+ public static String missingClassParseRequest;
+
+ /**
+ {
+ "sensorParserConfig":
+ {
+ "parserClassName": "badClass",
+ "sensorTopic": "squidTest",
+ "parserConfig": {
+ "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}",
+ "patternLabel": "SQUIDTEST",
+ "grokPath":"./squidTest",
+ "timestampField": "timestamp"
+ }
+ },
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html"
+ }
+ */
+ @Multiline
+ public static String badClassParseRequest;
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String sensorParserConfigUrl = "/api/v1/sensor/parser/config";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(sensorParserConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ cleanFileSystem();
+
+ this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser"))
+ .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
+ .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
+ .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
+ .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
+ .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
+ .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
+ .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
+ .andExpect(jsonPath("$.fieldTransformations[0].output[1]").value("domain_without_subdomains"))
+ .andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)"))
+ .andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"));
+
+ this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingGrokJson))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("A grokStatement must be provided"));
+
+ this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser"))
+ .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
+ .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
+ .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST"))
+ .andExpect(jsonPath("$.parserConfig.grokStatement").value("%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}"))
+ .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp"))
+ .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR"))
+ .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname"))
+ .andExpect(jsonPath("$.fieldTransformations[0].output[1]").value("domain_without_subdomains"))
+ .andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)"))
+ .andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"));
+
+ this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" +
+ "@.sensorTopic == 'squidTest' &&" +
+ "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" +
+ "@.parserConfig.patternLabel == 'SQUIDTEST' &&" +
+ "@.parserConfig.timestampField == 'timestamp' &&" +
+ "@.fieldTransformations[0].transformation == 'STELLAR' &&" +
+ "@.fieldTransformations[0].output[0] == 'full_hostname' &&" +
+ "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" +
+ "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" +
+ "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists());
+
+ this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser"))
+ .andExpect(jsonPath("$.sensorTopic").value("broTest"))
+ .andExpect(jsonPath("$.parserConfig").isEmpty());
+
+ this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser"))
+ .andExpect(jsonPath("$.sensorTopic").value("broTest"))
+ .andExpect(jsonPath("$.parserConfig").isEmpty());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" +
+ "@.sensorTopic == 'squidTest' &&" +
+ "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" +
+ "@.parserConfig.patternLabel == 'SQUIDTEST' &&" +
+ "@.parserConfig.timestampField == 'timestamp' &&" +
+ "@.fieldTransformations[0].transformation == 'STELLAR' &&" +
+ "@.fieldTransformations[0].output[0] == 'full_hostname' &&" +
+ "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" +
+ "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" +
+ "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists())
+ .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.bro.BasicBroParser' && " +
+ "@.sensorTopic == 'broTest')]").exists());
+
+ this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist())
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").exists());
+
+ this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist())
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+
+ this.mockMvc.perform(get(sensorParserConfigUrl + "/list/available").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser"))
+ .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser"));
+
+ this.mockMvc.perform(get(sensorParserConfigUrl + "/reload/available").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser"))
+ .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser"));
+
+ this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(parseRequest))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.elapsed").value(415))
+ .andExpect(jsonPath("$.code").value(200))
+ .andExpect(jsonPath("$.ip_dst_addr").value("207.109.73.154"))
+ .andExpect(jsonPath("$.method").value("GET"))
+ .andExpect(jsonPath("$.bytes").value(337891))
+ .andExpect(jsonPath("$.action").value("TCP_MISS"))
+ .andExpect(jsonPath("$.ip_src_addr").value("127.0.0.1"))
+ .andExpect(jsonPath("$.url").value("http://www.aliexpress.com/af/shoes.html?"))
+ .andExpect(jsonPath("$.timestamp").value(1467011157401L));
+
+ this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingConfigParseRequest))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("SensorParserConfig is missing from ParseMessageRequest"));
+
+ this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingClassParseRequest))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("SensorParserConfig must have a parserClassName"));
+
+ this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(badClassParseRequest))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("java.lang.ClassNotFoundException: badClass"));
+
+ }
+
+ private void cleanFileSystem() throws IOException {
+ File grokTempPath = new File(environment.getProperty(MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY));
+ if (grokTempPath.exists()) {
+ FileUtils.cleanDirectory(grokTempPath);
+ FileUtils.deleteDirectory(grokTempPath);
+ }
+ File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY));
+ if (grokPath.exists()) {
+ FileUtils.cleanDirectory(grokPath);
+ FileUtils.deleteDirectory(grokPath);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
new file mode 100644
index 0000000..c592159
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.env.Environment;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasSize;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class StormControllerIntegrationTest {
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ @Autowired
+ private GlobalConfigService globalConfigService;
+
+ @Autowired
+ private SensorParserConfigService sensorParserConfigService;
+
+ private MockMvc mockMvc;
+
+ private String stormUrl = "/api/v1/storm";
+ private String user = "user";
+ private String password = "password";
+
+ private String metronVersion;
+
+ @Before
+ public void setup() throws Exception {
+ this.metronVersion = this.environment.getProperty("metron.version");
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(get(stormUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/parser/start/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get("/enrichment"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/start"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/stop"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/activate"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get("/indexing"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/start"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/stop"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/activate"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/deactivate"))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$", hasSize(0)));
+
+ this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ Map<String, Object> globalConfig = globalConfigService.get();
+ if (globalConfig == null) {
+ globalConfig = new HashMap<>();
+ }
+ globalConfigService.delete();
+ sensorParserConfigService.delete("broTest");
+
+ this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.GLOBAL_CONFIG_MISSING.name()));
+
+ globalConfigService.save(globalConfig);
+
+ this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.name()));
+
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
+ sensorParserConfig.setSensorTopic("broTest");
+ sensorParserConfigService.save(sensorParserConfig);
+
+ this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value("broTest"))
+ .andExpect(jsonPath("$.id", containsString("broTest")))
+ .andExpect(jsonPath("$.status").value("ACTIVE"))
+ .andExpect(jsonPath("$.latency").exists())
+ .andExpect(jsonPath("$.throughput").exists());
+
+ this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.name == 'broTest' && @.status == 'ACTIVE')]").exists());
+
+ this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/stop?stopNow=true").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/start").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value("enrichment"))
+ .andExpect(jsonPath("$.id", containsString("enrichment")))
+ .andExpect(jsonPath("$.status").value("ACTIVE"))
+ .andExpect(jsonPath("$.latency").exists())
+ .andExpect(jsonPath("$.throughput").exists());
+
+ this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.name == 'enrichment' && @.status == 'ACTIVE')]").exists());
+
+ this.mockMvc.perform(get(stormUrl + "/enrichment/stop").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/stop?stopNow=true").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/start").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value("indexing"))
+ .andExpect(jsonPath("$.id", containsString("indexing")))
+ .andExpect(jsonPath("$.status").value("ACTIVE"))
+ .andExpect(jsonPath("$.latency").exists())
+ .andExpect(jsonPath("$.throughput").exists());
+
+ this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[?(@.name == 'indexing' && @.status == 'ACTIVE')]").exists());
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/stop").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/client/status").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.stormClientVersionInstalled").value("1.0.1"))
+ .andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh"))
+ .andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh"))
+ .andExpect(jsonPath("$.indexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
new file mode 100644
index 0000000..ec987e0
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/TransformationControllerIntegrationTest.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class TransformationControllerIntegrationTest {
+
+ private String valid = "TO_LOWER(test)";
+ private String invalid = "BAD_FUNCTION(test)";
+ private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]";
+
+ /**
+ {
+ "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]},
+ "sampleData": {"url": "https://caseystella.com/blog"}
+ }
+ */
+ @Multiline
+ public static String sensorParseConfigJson;
+
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String transformationUrl = "/api/v1/transformation";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(post(transformationUrl + "/validate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(transformationUrl + "/list"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(transformationUrl + "/list/functions"))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(post(transformationUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE))
+ .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE));
+
+ this.mockMvc.perform(post(transformationUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseConfigJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.url").value("https://caseystella.com/blog"))
+ .andExpect(jsonPath("$.url_host").value("caseystella.com"));
+
+ this.mockMvc.perform(get(transformationUrl + "/list").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+ this.mockMvc.perform(get(transformationUrl + "/list/functions").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+
+ this.mockMvc.perform(get(transformationUrl + "/list/simple/functions").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$", hasSize(greaterThan(0))));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java
new file mode 100644
index 0000000..9bdd439
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class UserControllerIntegrationTest {
+
+ private MockMvc mockMvc;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(get("/api/v1/user"))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get("/api/v1/user").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().string(user));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java
new file mode 100644
index 0000000..8557035
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.generator;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.FileUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SampleDataGenerator {
+
+ private Logger LOG = LoggerFactory.getLogger(SampleDataGenerator.class);
+
+ private Map<String, List<String>> sampleData = new HashMap<>();
+ private Map<String, Integer> indexes = new HashMap<>();
+ private KafkaProducer kafkaProducer;
+ private String brokerUrl;
+ private Integer num = -1;
+ private String selectedSensorType = null;
+ private Integer delay = 1000;
+
+ public void setBrokerUrl(String brokerUrl) {
+ this.brokerUrl = brokerUrl;
+ }
+
+ public void setNum(Integer num) {
+ this.num = num;
+ }
+
+ public void setSelectedSensorType(String selectedSensorType) {
+ this.selectedSensorType = selectedSensorType;
+ }
+
+ public void setDelay(Integer delay) {
+ this.delay = delay;
+ }
+
+ public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException, ParseException {
+ CommandLineParser parser = new PosixParser();
+ CommandLine cli = parser.parse(getOptions(), args);
+ Integer num = Integer.parseInt(cli.getOptionValue("n", "-1"));
+ String selectedSensorType = cli.getOptionValue("s");
+ Integer delay = Integer.parseInt(cli.getOptionValue("d", "1000"));
+ String path = cli.getOptionValue("p");
+ if (selectedSensorType == null || path == null) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp( "sample_data_generator", getOptions());
+ } else {
+ SampleDataGenerator sampleDataGenerator = new SampleDataGenerator();
+ sampleDataGenerator.setNum(num);
+ sampleDataGenerator.setSelectedSensorType(selectedSensorType);
+ sampleDataGenerator.setDelay(delay);
+ sampleDataGenerator.generateSampleData(path);
+ }
+
+ }
+
+ private static Options getOptions() {
+ Options options = new Options();
+ options.addOption("b", "brokerUrl", true, "Kafka Broker Url");
+ options.addOption("n", "num", false, "Number of messages to emit");
+ options.addOption("s", "sensorType", true, "Emit messages to this topic");
+ options.addOption("d", "delay", false, "Number of milliseconds to wait between each message. Defaults to 1 second");
+ options.addOption("p", "path", true, "Local path to data file");
+ return options;
+ }
+
+ public void generateSampleData(String path) throws IOException, ParseException {
+ loadData(path);
+ startClients();
+ try {
+ emitData(num, selectedSensorType, delay);
+ } finally {
+ stopClients();
+ }
+ }
+
+ private void loadData(String sampleDataPath) throws IOException, ParseException {
+ sampleData.put(selectedSensorType, FileUtils.readLines(new File(sampleDataPath)));
+ indexes.put(selectedSensorType, 0);
+ }
+
+ private void emitData(int num, String selectedSensorType, int delay) {
+ int count = 0;
+ boolean continueEmitting = false;
+ do {
+ for (String sensorType : sampleData.keySet()) {
+ if (selectedSensorType == null || selectedSensorType.equals(sensorType)) {
+ List<String> sensorData = sampleData.get(sensorType);
+ int index = indexes.get(sensorType);
+ String message = sensorData.get(index++);
+ emitSensorData(sensorType, message, delay);
+ if (num != -1 && ++count >= num) {
+ continueEmitting = false;
+ break;
+ }
+ continueEmitting = true;
+ if (index == sensorData.size()) {
+ index = 0;
+ }
+ indexes.put(sensorType, index);
+ }
+ }
+ } while (continueEmitting);
+ }
+
+ private void emitSensorData(String sensorType, String message, int delay) {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ LOG.info("Emitting " + sensorType + " message " + message);
+ emitToKafka(sensorType, message);
+ }
+
+ private void startClients() {
+ startKafka();
+ }
+
+ private void startKafka() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", brokerUrl);
+ producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ kafkaProducer = new KafkaProducer<>(producerConfig);
+ }
+
+ private void stopClients() {
+ stopKafka();
+ }
+
+ private void stopKafka() {
+ LOG.info("Stopping Kafka producer");
+ kafkaProducer.close();
+ }
+
+ private void emitToKafka(String topic, String message) {
+ kafkaProducer.send(new ProducerRecord<String, String>(topic, message));
+ }
+
+}
[05/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
new file mode 100644
index 0000000..1166e9c
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.io.InputStreamReader;
+
+@Configuration
+public class GrokConfig {
+
+ @Bean
+ public Grok commonGrok() throws GrokException {
+ Grok grok = new Grok();
+ grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
+ return grok;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
new file mode 100644
index 0000000..f4b8bdd
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.apache.metron.rest.MetronRestConstants;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
+
+@Configuration
+public class HadoopConfig {
+
+ @Autowired
+ private Environment environment;
+
+ @Bean
+ public org.apache.hadoop.conf.Configuration configuration() {
+ org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
+ configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL));
+ return configuration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
new file mode 100644
index 0000000..8044405
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import kafka.utils.ZkUtils;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.metron.rest.MetronRestConstants;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.env.Environment;
+
+import java.util.Properties;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile("!" + TEST_PROFILE)
+public class KafkaConfig {
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private ZkClient zkClient;
+
+ @Bean
+ public ZkUtils zkUtils() {
+ return ZkUtils.apply(zkClient, false);
+ }
+
+ @Bean(destroyMethod="close")
+ public KafkaConsumer<String, String> kafkaConsumer() {
+ Properties props = new Properties();
+ props.put("bootstrap.servers", environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY));
+ props.put("group.id", "metron-config");
+ props.put("enable.auto.commit", "false");
+ props.put("auto.commit.interval.ms", "1000");
+ props.put("session.timeout.ms", "30000");
+ props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ return new KafkaConsumer<>(props);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java
new file mode 100644
index 0000000..ed82164
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.CorsRegistry;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
+
+@Configuration
+public class MvcConfig extends WebMvcConfigurerAdapter {
+
+ @Override
+ public void addCorsMappings(CorsRegistry registry) {
+ registry.addMapping("/**");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java
new file mode 100644
index 0000000..8fea90c
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.web.client.RestTemplate;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile("!" + TEST_PROFILE)
+public class RestTemplateConfig {
+
+ @Bean
+ public RestTemplate restTemplate() {
+ return new RestTemplate();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
new file mode 100644
index 0000000..d6d0cff
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.apache.metron.rest.service.impl.DockerStormCLIWrapper;
+import org.apache.metron.rest.service.impl.StormCLIWrapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.env.Environment;
+
+import java.util.Arrays;
+
+import static org.apache.metron.rest.MetronRestConstants.DOCKER_PROFILE;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile("!" + TEST_PROFILE)
+public class StormConfig {
+
+ @Autowired
+ private Environment environment;
+
+ @Bean
+ public StormCLIWrapper stormCLIClientWrapper() {
+ if (Arrays.asList(environment.getActiveProfiles()).contains(DOCKER_PROFILE)) {
+ return new DockerStormCLIWrapper();
+ } else {
+ return new StormCLIWrapper();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
new file mode 100644
index 0000000..9e82196
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerConfig {
+ @Bean
+ public Docket api() {
+ return new Docket(DocumentationType.SWAGGER_2)
+ .select()
+ .apis(RequestHandlerSelectors.any())
+ .paths(PathSelectors.any())
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java
new file mode 100644
index 0000000..789098a
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.apache.metron.rest.MetronRestConstants;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.web.authentication.logout.HttpStatusReturningLogoutSuccessHandler;
+import org.springframework.security.web.csrf.CookieCsrfTokenRepository;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+import javax.sql.DataSource;
+import java.util.Arrays;
+import java.util.List;
+
+@Configuration
+@EnableWebSecurity
+@Controller
+public class WebSecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Autowired
+ private Environment environment;
+
+ @RequestMapping({"/login", "/logout", "/sensors", "/sensors*/**"})
+ public String handleNGRequests() {
+ return "forward:/index.html";
+ }
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http
+ .authorizeRequests()
+ .antMatchers("/", "/home", "/login").permitAll()
+ .antMatchers("/app/**").permitAll()
+ .antMatchers("/vendor/**").permitAll()
+ .antMatchers("/fonts/**").permitAll()
+ .antMatchers("/assets/images/**").permitAll()
+ .antMatchers("/*.js").permitAll()
+ .antMatchers("/*.ttf").permitAll()
+ .antMatchers("/*.woff2").permitAll()
+ .anyRequest().authenticated()
+ .and().httpBasic()
+ .and()
+ .logout()
+ .logoutSuccessHandler(new HttpStatusReturningLogoutSuccessHandler())
+ .invalidateHttpSession(true)
+ .deleteCookies("JSESSIONID");
+ if (Arrays.asList(environment.getActiveProfiles()).contains(MetronRestConstants.CSRF_ENABLE_PROFILE)) {
+ http.csrf().csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse());
+ } else {
+ http.csrf().disable();
+ }
+ }
+
+ @Autowired
+ private DataSource dataSource;
+
+ @Autowired
+ public void configureJdbc(AuthenticationManagerBuilder auth) throws Exception {
+ List<String> activeProfiles = Arrays.asList(environment.getActiveProfiles());
+ if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) ||
+ activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) {
+ auth.jdbcAuthentication().dataSource(dataSource)
+ .withUser("user").password("password").roles("USER").and()
+ .withUser("user1").password("password").roles("USER").and()
+ .withUser("user2").password("password").roles("USER").and()
+ .withUser("admin").password("password").roles("USER", "ADMIN");
+ } else {
+ auth.jdbcAuthentication().dataSource(dataSource);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java
new file mode 100644
index 0000000..5f58193
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import kafka.utils.ZKStringSerializer$;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.metron.rest.MetronRestConstants;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.env.Environment;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile("!" + TEST_PROFILE)
+public class ZookeeperConfig {
+
+ @Bean(initMethod = "start", destroyMethod="close")
+ public CuratorFramework client(Environment environment) {
+ int sleepTime = Integer.parseInt(environment.getProperty(MetronRestConstants.CURATOR_SLEEP_TIME));
+ int maxRetries = Integer.parseInt(environment.getProperty(MetronRestConstants.CURATOR_MAX_RETRIES));
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(sleepTime, maxRetries);
+ return CuratorFrameworkFactory.newClient(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY), retryPolicy);
+ }
+
+ @Bean(destroyMethod="close")
+ public ZkClient zkClient(Environment environment) {
+ int sessionTimeout = Integer.parseInt(environment.getProperty(MetronRestConstants.ZK_CLIENT_SESSION_TIMEOUT));
+ int connectionTimeout = Integer.parseInt(environment.getProperty(MetronRestConstants.ZK_CLIENT_CONNECTION_TIMEOUT));
+ return new ZkClient(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY), sessionTimeout, connectionTimeout, ZKStringSerializer$.MODULE$);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java
new file mode 100644
index 0000000..739d68b
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/global/config")
+public class GlobalConfigController {
+
+ @Autowired
+ private GlobalConfigService globalConfigService;
+
+ @ApiOperation(value = "Creates or updates the Global Config in Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Global Config updated. Returns saved Global Config JSON", code = 200),
+ @ApiResponse(message = "Global Config created. Returns saved Global Config JSON", code = 201) })
+ @RequestMapping(method = RequestMethod.POST)
+ ResponseEntity<Map<String, Object>> save(@ApiParam(name="globalConfig", value="The Global Config JSON to be saved", required=true)@RequestBody Map<String, Object> globalConfig) throws RestException {
+ if (globalConfigService.get() == null) {
+ return new ResponseEntity<>(globalConfigService.save(globalConfig), HttpStatus.CREATED);
+ } else {
+ return new ResponseEntity<>(globalConfigService.save(globalConfig), HttpStatus.OK);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves the current Global Config from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Returns current Global Config JSON in Zookeeper", code = 200),
+ @ApiResponse(message = "Global Config JSON was not found in Zookeeper", code = 404) })
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<Map<String, Object>> get() throws RestException {
+ Map<String, Object> globalConfig = globalConfigService.get();
+ if (globalConfig != null) {
+ return new ResponseEntity<>(globalConfig, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Deletes the current Global Config from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Global Config JSON was deleted", code = 200),
+ @ApiResponse(message = "Global Config JSON was not found in Zookeeper", code = 404) })
+ @RequestMapping(method = RequestMethod.DELETE)
+ ResponseEntity<Void> delete() throws RestException {
+ if (globalConfigService.delete()) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
new file mode 100644
index 0000000..2b155b1
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.GrokValidation;
+import org.apache.metron.rest.service.GrokService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/grok")
+public class GrokController {
+
+ @Autowired
+ private GrokService grokService;
+
+ @ApiOperation(value = "Applies a Grok statement to a sample message")
+ @ApiResponse(message = "JSON results", code = 200)
+ @RequestMapping(value = "/validate", method = RequestMethod.POST)
+ ResponseEntity<GrokValidation> post(@ApiParam(name="grokValidation", value="Object containing Grok statment and sample message", required=true)@RequestBody GrokValidation grokValidation) throws RestException {
+ return new ResponseEntity<>(grokService.validateGrokStatement(grokValidation), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the common Grok statements available in Metron")
+ @ApiResponse(message = "JSON object containing pattern label/Grok statements key value pairs", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<Map<String, String>> list() throws RestException {
+ return new ResponseEntity<>(grokService.getCommonGrokPatterns(), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
new file mode 100644
index 0000000..876558b
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.KafkaTopic;
+import org.apache.metron.rest.service.KafkaService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Set;
+
+@RestController
+@RequestMapping("/api/v1/kafka")
+public class KafkaController {
+
+ @Autowired
+ private KafkaService kafkaService;
+
+ @ApiOperation(value = "Creates a new Kafka topic")
+ @ApiResponse(message = "Returns saved Kafka topic", code = 200)
+ @RequestMapping(value = "/topic", method = RequestMethod.POST)
+ ResponseEntity<KafkaTopic> save(@ApiParam(name="topic", value="Kafka topic", required=true)@RequestBody KafkaTopic topic) throws RestException {
+ return new ResponseEntity<>(kafkaService.createTopic(topic), HttpStatus.CREATED);
+ }
+
+ @ApiOperation(value = "Retrieves a Kafka topic")
+ @ApiResponses(value = { @ApiResponse(message = "Returns Kafka topic", code = 200),
+ @ApiResponse(message = "Kafka topic is missing", code = 404) })
+ @RequestMapping(value = "/topic/{name}", method = RequestMethod.GET)
+ ResponseEntity<KafkaTopic> get(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException {
+ KafkaTopic kafkaTopic = kafkaService.getTopic(name);
+ if (kafkaTopic != null) {
+ return new ResponseEntity<>(kafkaTopic, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves all Kafka topics")
+ @ApiResponse(message = "Returns a list of all Kafka topics", code = 200)
+ @RequestMapping(value = "/topic", method = RequestMethod.GET)
+ ResponseEntity<Set<String>> list() throws Exception {
+ return new ResponseEntity<>(kafkaService.listTopics(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Delets a Kafka topic")
+ @ApiResponses(value = { @ApiResponse(message = "Kafka topic was deleted", code = 200),
+ @ApiResponse(message = "Kafka topic is missing", code = 404) })
+ @RequestMapping(value = "/topic/{name}", method = RequestMethod.DELETE)
+ ResponseEntity<Void> delete(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException {
+ if (kafkaService.deleteTopic(name)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves a sample message from a Kafka topic using the most recent offset")
+ @ApiResponses(value = { @ApiResponse(message = "Returns sample message", code = 200),
+ @ApiResponse(message = "Either Kafka topic is missing or contains no messages", code = 404) })
+ @RequestMapping(value = "/topic/{name}/sample", method = RequestMethod.GET)
+ ResponseEntity<String> getSample(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException {
+ String sampleMessage = kafkaService.getSampleMessage(name);
+ if (sampleMessage != null) {
+ return new ResponseEntity<>(sampleMessage, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java
new file mode 100644
index 0000000..fa73dab
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.RestError;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
+
+import javax.servlet.http.HttpServletRequest;
+
+@ControllerAdvice(basePackages = "org.apache.metron.rest.controller")
+public class RestExceptionHandler extends ResponseEntityExceptionHandler {
+
+ @ExceptionHandler(RestException.class)
+ @ResponseBody
+ ResponseEntity<?> handleControllerException(HttpServletRequest request, Throwable ex) {
+ HttpStatus status = getStatus(request);
+ return new ResponseEntity<>(new RestError(status.value(), ex.getMessage(), ex.getMessage()), status);
+ }
+
+ private HttpStatus getStatus(HttpServletRequest request) {
+ Integer statusCode = (Integer) request.getAttribute("javax.servlet.error.status_code");
+ if (statusCode == null) {
+ return HttpStatus.INTERNAL_SERVER_ERROR;
+ }
+ return HttpStatus.valueOf(statusCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java
new file mode 100644
index 0000000..546384a
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorEnrichmentConfigService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/sensor/enrichment/config")
+public class SensorEnrichmentConfigController {
+
+ @Autowired
+ private SensorEnrichmentConfigService sensorEnrichmentConfigService;
+
+ @ApiOperation(value = "Updates or creates a SensorEnrichmentConfig in Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorEnrichmentConfig updated. Returns saved SensorEnrichmentConfig", code = 200),
+ @ApiResponse(message = "SensorEnrichmentConfig created. Returns saved SensorEnrichmentConfig", code = 201) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.POST)
+ ResponseEntity<SensorEnrichmentConfig> save(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name,
+ @ApiParam(name="sensorEnrichmentConfig", value="SensorEnrichmentConfig", required=true)@RequestBody SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
+ if (sensorEnrichmentConfigService.findOne(name) == null) {
+ return new ResponseEntity<>(sensorEnrichmentConfigService.save(name, sensorEnrichmentConfig), HttpStatus.CREATED);
+ } else {
+ return new ResponseEntity<>(sensorEnrichmentConfigService.save(name, sensorEnrichmentConfig), HttpStatus.OK);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves a SensorEnrichmentConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Returns SensorEnrichmentConfig", code = 200),
+ @ApiResponse(message = "SensorEnrichmentConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.GET)
+ ResponseEntity<SensorEnrichmentConfig> findOne(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name) throws RestException {
+ SensorEnrichmentConfig sensorEnrichmentConfig = sensorEnrichmentConfigService.findOne(name);
+ if (sensorEnrichmentConfig != null) {
+ return new ResponseEntity<>(sensorEnrichmentConfig, HttpStatus.OK);
+ }
+
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+
+ @ApiOperation(value = "Retrieves all SensorEnrichmentConfigs from Zookeeper")
+ @ApiResponse(message = "Returns all SensorEnrichmentConfigs", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<Map<String, SensorEnrichmentConfig>> getAll() throws Exception {
+ return new ResponseEntity<>(sensorEnrichmentConfigService.getAll(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deletes a SensorEnrichmentConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorEnrichmentConfig was deleted", code = 200),
+ @ApiResponse(message = "SensorEnrichmentConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.DELETE)
+ ResponseEntity<Void> delete(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name) throws RestException {
+ if (sensorEnrichmentConfigService.delete(name)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Lists the available enrichments")
+ @ApiResponse(message = "Returns a list of available enrichments", code = 200)
+ @RequestMapping(value = "/list/available", method = RequestMethod.GET)
+ ResponseEntity<List<String>> getAvailable() throws RestException {
+ return new ResponseEntity<>(sensorEnrichmentConfigService.getAvailableEnrichments(), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
new file mode 100644
index 0000000..ffd80fb
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorIndexingConfigService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/sensor/indexing/config")
+public class SensorIndexingConfigController {
+
+ @Autowired
+ private SensorIndexingConfigService sensorIndexingConfigService;
+
+ @ApiOperation(value = "Updates or creates a SensorIndexingConfig in Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig updated. Returns saved SensorIndexingConfig", code = 200),
+ @ApiResponse(message = "SensorIndexingConfig created. Returns saved SensorIndexingConfig", code = 201) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Object>> save(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name,
+ @ApiParam(name="sensorIndexingConfig", value="SensorIndexingConfig", required=true)@RequestBody Map<String, Object> sensorIndexingConfig) throws RestException {
+ if (sensorIndexingConfigService.findOne(name) == null) {
+ return new ResponseEntity<>(sensorIndexingConfigService.save(name, sensorIndexingConfig), HttpStatus.CREATED);
+ } else {
+ return new ResponseEntity<>(sensorIndexingConfigService.save(name, sensorIndexingConfig), HttpStatus.OK);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves a SensorIndexingConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Returns SensorIndexingConfig", code = 200),
+ @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.GET)
+ ResponseEntity<Map<String, Object>> findOne(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name) throws RestException {
+ Map<String, Object> sensorIndexingConfig = sensorIndexingConfigService.findOne(name);
+ if (sensorIndexingConfig != null) {
+ return new ResponseEntity<>(sensorIndexingConfig, HttpStatus.OK);
+ }
+
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+
+ @ApiOperation(value = "Retrieves all SensorIndexingConfigs from Zookeeper")
+ @ApiResponse(message = "Returns all SensorIndexingConfigs", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<Map<String, Map<String, Object>>> getAll() throws Exception {
+ return new ResponseEntity<>(sensorIndexingConfigService.getAll(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deletes a SensorIndexingConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig was deleted", code = 200),
+ @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.DELETE)
+ ResponseEntity<Void> delete(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name) throws RestException {
+ if (sensorIndexingConfigService.delete(name)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java
new file mode 100644
index 0000000..189e2af
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.ParseMessageRequest;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.json.simple.JSONObject;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/sensor/parser/config")
+public class SensorParserConfigController {
+
+ @Autowired
+ private SensorParserConfigService sensorParserConfigService;
+
+ @ApiOperation(value = "Updates or creates a SensorParserConfig in Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorParserConfig updated. Returns saved SensorParserConfig", code = 200),
+ @ApiResponse(message = "SensorParserConfig created. Returns saved SensorParserConfig", code = 201) })
+ @RequestMapping(method = RequestMethod.POST)
+ ResponseEntity<SensorParserConfig> save(@ApiParam(name="sensorParserConfig", value="SensorParserConfig", required=true)@RequestBody SensorParserConfig sensorParserConfig) throws RestException {
+ String name = sensorParserConfig.getSensorTopic();
+ if (sensorParserConfigService.findOne(name) == null) {
+ return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.CREATED);
+ } else {
+ return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.OK);
+ }
+ }
+
+ @ApiOperation(value = "Retrieves a SensorParserConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "Returns SensorParserConfig", code = 200),
+ @ApiResponse(message = "SensorParserConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.GET)
+ ResponseEntity<SensorParserConfig> findOne(@ApiParam(name="name", value="SensorParserConfig name", required=true)@PathVariable String name) throws RestException {
+ SensorParserConfig sensorParserConfig = sensorParserConfigService.findOne(name);
+ if (sensorParserConfig != null) {
+ return new ResponseEntity<>(sensorParserConfig, HttpStatus.OK);
+ }
+
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+
+ @ApiOperation(value = "Retrieves all SensorParserConfigs from Zookeeper")
+ @ApiResponse(message = "Returns all SensorParserConfigs", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<Iterable<SensorParserConfig>> findAll() throws RestException {
+ return new ResponseEntity<>(sensorParserConfigService.getAll(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deletes a SensorParserConfig from Zookeeper")
+ @ApiResponses(value = { @ApiResponse(message = "SensorParserConfig was deleted", code = 200),
+ @ApiResponse(message = "SensorParserConfig is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.DELETE)
+ ResponseEntity<Void> delete(@ApiParam(name="name", value="SensorParserConfig name", required=true)@PathVariable String name) throws RestException {
+ if (sensorParserConfigService.delete(name)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Lists the available parser classes that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of available parser classes", code = 200)
+ @RequestMapping(value = "/list/available", method = RequestMethod.GET)
+ ResponseEntity<Map<String, String>> getAvailable() throws RestException {
+ return new ResponseEntity<>(sensorParserConfigService.getAvailableParsers(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Scans the classpath for available parser classes and reloads the cached parser class list")
+ @ApiResponse(message = "Returns a list of available parser classes", code = 200)
+ @RequestMapping(value = "/reload/available", method = RequestMethod.GET)
+ ResponseEntity<Map<String, String>> reloadAvailable() throws RestException {
+ return new ResponseEntity<>(sensorParserConfigService.reloadAvailableParsers(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Parses a sample message given a SensorParserConfig")
+ @ApiResponse(message = "Returns parsed message", code = 200)
+ @RequestMapping(value = "/parseMessage", method = RequestMethod.POST)
+ ResponseEntity<JSONObject> parseMessage(@ApiParam(name="parseMessageRequest", value="Object containing a sample message and SensorParserConfig", required=true) @RequestBody ParseMessageRequest parseMessageRequest) throws RestException {
+ return new ResponseEntity<>(sensorParserConfigService.parseMessage(parseMessageRequest), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
new file mode 100644
index 0000000..915e304
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatus;
+import org.apache.metron.rest.service.StormAdminService;
+import org.apache.metron.rest.service.StormStatusService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/storm")
+public class StormController {
+
+ @Autowired
+ private StormStatusService stormStatusService;
+
+ @Autowired
+ private StormAdminService stormAdminService;
+
+ @ApiOperation(value = "Retrieves the status of all Storm topologies")
+ @ApiResponse(message = "Returns a list of topologies with status information", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<List<TopologyStatus>> getAll() throws RestException {
+ return new ResponseEntity<>(stormStatusService.getAllTopologyStatus(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves the status of a Storm topology")
+ @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200),
+ @ApiResponse(message = "Topology is missing", code = 404) })
+ @RequestMapping(value = "/{name}", method = RequestMethod.GET)
+ ResponseEntity<TopologyStatus> get(@ApiParam(name="name", value="Topology name", required=true)@PathVariable String name) throws RestException {
+ TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(name);
+ if (topologyStatus != null) {
+ return new ResponseEntity<>(topologyStatus, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Starts a Storm parser topology")
+ @ApiResponse(message = "Returns start response message", code = 200)
+ @RequestMapping(value = "/parser/start/{name}", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> start(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException {
+ return new ResponseEntity<>(stormAdminService.startParserTopology(name), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Stops a Storm parser topology")
+ @ApiResponse(message = "Returns stop response message", code = 200)
+ @RequestMapping(value = "/parser/stop/{name}", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> stop(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name,
+ @ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
+ return new ResponseEntity<>(stormAdminService.stopParserTopology(name, stopNow), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Activates a Storm parser topology")
+ @ApiResponse(message = "Returns activate response message", code = 200)
+ @RequestMapping(value = "/parser/activate/{name}", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> activate(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException {
+ return new ResponseEntity<>(stormStatusService.activateTopology(name), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deactivates a Storm parser topology")
+ @ApiResponse(message = "Returns deactivate response message", code = 200)
+ @RequestMapping(value = "/parser/deactivate/{name}", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> deactivate(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException {
+ return new ResponseEntity<>(stormStatusService.deactivateTopology(name), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves the status of the Storm enrichment topology")
+ @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200),
+ @ApiResponse(message = "Topology is missing", code = 404) })
+ @RequestMapping(value = "/enrichment", method = RequestMethod.GET)
+ ResponseEntity<TopologyStatus> getEnrichment() throws RestException {
+ TopologyStatus sensorParserStatus = stormStatusService.getTopologyStatus(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME);
+ if (sensorParserStatus != null) {
+ return new ResponseEntity<>(sensorParserStatus, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Starts a Storm enrichment topology")
+ @ApiResponse(message = "Returns start response message", code = 200)
+ @RequestMapping(value = "/enrichment/start", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> startEnrichment() throws RestException {
+ return new ResponseEntity<>(stormAdminService.startEnrichmentTopology(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Stops a Storm enrichment topology")
+ @ApiResponse(message = "Returns stop response message", code = 200)
+ @RequestMapping(value = "/enrichment/stop", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> stopEnrichment(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
+ return new ResponseEntity<>(stormAdminService.stopEnrichmentTopology(stopNow), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Activates a Storm enrichment topology")
+ @ApiResponse(message = "Returns activate response message", code = 200)
+ @RequestMapping(value = "/enrichment/activate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> activateEnrichment() throws RestException {
+ return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deactivates a Storm enrichment topology")
+ @ApiResponse(message = "Returns deactivate response message", code = 200)
+ @RequestMapping(value = "/enrichment/deactivate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> deactivateEnrichment() throws RestException {
+ return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves the status of the Storm indexing topology")
+ @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200),
+ @ApiResponse(message = "Topology is missing", code = 404) })
+ @RequestMapping(value = "/indexing", method = RequestMethod.GET)
+ ResponseEntity<TopologyStatus> getIndexing() throws RestException {
+ TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(MetronRestConstants.INDEXING_TOPOLOGY_NAME);
+ if (topologyStatus != null) {
+ return new ResponseEntity<>(topologyStatus, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Starts a Storm indexing topology")
+ @ApiResponse(message = "Returns start response message", code = 200)
+ @RequestMapping(value = "/indexing/start", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> startIndexing() throws RestException {
+ return new ResponseEntity<>(stormAdminService.startIndexingTopology(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Stops a Storm enrichment topology")
+ @ApiResponse(message = "Returns stop response message", code = 200)
+ @RequestMapping(value = "/indexing/stop", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> stopIndexing(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
+ return new ResponseEntity<>(stormAdminService.stopIndexingTopology(stopNow), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Activates a Storm indexing topology")
+ @ApiResponse(message = "Returns activate response message", code = 200)
+ @RequestMapping(value = "/indexing/activate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> activateIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deactivates a Storm indexing topology")
+ @ApiResponse(message = "Returns deactivate response message", code = 200)
+ @RequestMapping(value = "/indexing/deactivate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> deactivateIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves information about the Storm command line client")
+ @ApiResponse(message = "Returns storm command line client information", code = 200)
+ @RequestMapping(value = "/client/status", method = RequestMethod.GET)
+ ResponseEntity<Map<String, String>> clientStatus() throws RestException {
+ return new ResponseEntity<>(stormAdminService.getStormClientStatus(), HttpStatus.OK);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
new file mode 100644
index 0000000..bc8b201
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.TransformationValidation;
+import org.apache.metron.rest.service.TransformationService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/transformation")
+public class TransformationController {
+
+ @Autowired
+ private TransformationService transformationService;
+
+ @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
+ @ApiResponse(message = "Returns validation results", code = 200)
+ @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Boolean>> validateRule(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
+ return new ResponseEntity<>(transformationService.validateRules(statements), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Executes transformations against a sample message")
+ @ApiResponse(message = "Returns transformation results", code = 200)
+ @RequestMapping(value = "/validate", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Object>> validateTransformation(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody TransformationValidation transformationValidation) throws RestException {
+ return new ResponseEntity<>(transformationService.validateTransformation(transformationValidation), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves field transformations")
+ @ApiResponse(message = "Returns a list field transformations", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<FieldTransformations[]> list() throws RestException {
+ return new ResponseEntity<>(transformationService.getTransformations(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
+ @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
+ return new ResponseEntity<>(transformationService.getStellarFunctions(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
+ @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
+ return new ResponseEntity<>(transformationService.getSimpleStellarFunctions(), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java
new file mode 100644
index 0000000..594b51f
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiResponse;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.security.Principal;
+
+@RestController
+public class UserController {
+
+ @ApiOperation(value = "Retrieves the current user")
+ @ApiResponse(message = "Current user", code = 200)
+ @RequestMapping("/api/v1/user")
+ public String user(Principal user) {
+ return user.getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java
new file mode 100644
index 0000000..7b8cf45
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+
+import java.util.Map;
+
+public interface GlobalConfigService {
+
+ Map<String, Object> save(Map<String, Object> globalConfig) throws RestException;
+
+ Map<String, Object> get() throws RestException;
+
+ boolean delete() throws RestException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
new file mode 100644
index 0000000..95ce1ba
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.GrokValidation;
+
+import java.util.Map;
+
+public interface GrokService {
+
+ Map<String, String> getCommonGrokPatterns();
+
+ GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
new file mode 100644
index 0000000..97888ff
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+
+public interface HdfsService {
+
+ byte[] read(Path path) throws IOException;
+
+ void write(Path path, byte[] contents) throws IOException;
+
+ FileStatus[] list(Path path) throws IOException;
+
+ boolean delete(Path path, boolean recursive) throws IOException;
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
new file mode 100644
index 0000000..e89b165
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.KafkaTopic;
+
+import java.util.Set;
+
+public interface KafkaService {
+
+ KafkaTopic createTopic(KafkaTopic topic) throws RestException;
+
+ boolean deleteTopic(String name);
+
+ KafkaTopic getTopic(String name);
+
+ Set<String> listTopics();
+
+ String getSampleMessage(String topic);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
new file mode 100644
index 0000000..fe84802
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.rest.RestException;
+
+import java.util.List;
+import java.util.Map;
+
+public interface SensorEnrichmentConfigService {
+
+ SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException;
+
+ SensorEnrichmentConfig findOne(String name) throws RestException;
+
+ Map<String, SensorEnrichmentConfig> getAll() throws RestException;
+
+ List<String> getAllTypes() throws RestException;
+
+ boolean delete(String name) throws RestException;
+
+ List<String> getAvailableEnrichments();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
new file mode 100644
index 0000000..0699431
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+
+import java.util.List;
+import java.util.Map;
+
+public interface SensorIndexingConfigService {
+
+ Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException;
+
+ Map<String, Object> findOne(String name) throws RestException;
+
+ Map<String, Map<String, Object>> getAll() throws RestException;
+
+ List<String> getAllTypes() throws RestException;
+
+ boolean delete(String name) throws RestException;
+
+}
[03/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE b/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000..c9d4b53
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,692 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------
+ DOM4J
+------------------------------------------------------------------------------------
+
+This product bundles dom4j 1.6.1. For details, see https://github.com/dom4j/dom4j.
+
+Copyright 2001-2016 (C) MetaStuff, Ltd. and DOM4J contributors. All Rights Reserved.
+
+Redistribution and use of this software and associated documentation
+("Software"), with or without modification, are permitted provided
+that the following conditions are met:
+
+1. Redistributions of source code must retain copyright
+ statements and notices. Redistributions must also contain a
+ copy of this document.
+
+2. Redistributions in binary form must reproduce the
+ above copyright notice, this list of conditions and the
+ following disclaimer in the documentation and/or other
+ materials provided with the distribution.
+
+3. The name "DOM4J" must not be used to endorse or promote
+ products derived from this Software without prior written
+ permission of MetaStuff, Ltd. For written permission,
+ please contact dom4j-info@metastuff.com.
+
+4. Products derived from this Software may not be called "DOM4J"
+ nor may "DOM4J" appear in their names without prior written
+ permission of MetaStuff, Ltd. DOM4J is a registered
+ trademark of MetaStuff, Ltd.
+
+5. Due credit should be given to the DOM4J Project - https://dom4j.github.io/
+
+THIS SOFTWARE IS PROVIDED BY METASTUFF, LTD. AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+METASTUFF, LTD. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------------
+ EPL
+------------------------------------------------------------------------------------
+
+This product bundles aspectjweaver 1.8.9, which is available under a "Eclipse Public License - v 1.0" license. For details, see https://eclipse.org/aspectj.
+
+Eclipse Public License - v 1.0
+
+THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT\u2019S ACCEPTANCE OF THIS AGREEMENT.
+
+1. DEFINITIONS
+
+"Contribution" means:
+
+a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and
+b) in the case of each subsequent Contributor:
+
+i)changes to the Program, and
+
+ii)additions to the Program;
+
+where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor\u2019s behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program.
+"Contributor" means any person or entity that distributes the Program.
+
+"Licensed Patents " mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program.
+
+"Program" means the Contributions distributed in accordance with this Agreement.
+
+"Recipient" means anyone who receives the Program under this Agreement, including all Contributors.
+
+2. GRANT OF RIGHTS
+
+a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form.
+
+b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder.
+
+c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient\u2019s responsibility to acquire that license before distributing the Program.
+
+d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement.
+
+3. REQUIREMENTS
+
+A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that:
+
+a) it complies with the terms and conditions of this Agreement; and
+
+b) its license agreement:
+
+i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose;
+
+ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits;
+
+iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and
+
+iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange.
+When the Program is made available in source code form:
+
+a) it must be made available under this Agreement; and
+
+b) a copy of this Agreement must be included with each copy of the Program.
+Contributors may not remove or alter any copyright notices contained within the Program.
+
+Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution.
+
+4. COMMERCIAL DISTRIBUTION
+
+Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Los
ses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense.
+
+For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor\u2019s responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages.
+
+5. NO WARRANTY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations.
+
+6. DISCLAIMER OF LIABILITY
+
+EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. GENERAL
+
+If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
+
+If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient\u2019s patent(s), then such Recipient\u2019s rights granted under Section 2(b) shall terminate as of the date such litigation is filed.
+
+All Recipient\u2019s rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient\u2019s rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient\u2019s obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive.
+
+Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b)
above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved.
+
+This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation.
+
+------------------------------------------------------------------------------------
+ BSD
+------------------------------------------------------------------------------------
+
+This product bundles antlr 2.7.7, which is available under a "BSD Software License" license. For details, see http://www.antlr2.org
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------------
+ CDDL v1.0
+------------------------------------------------------------------------------------
+
+This product bundles javax.transaction-api 1.2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://java.net/projects/jta-spec/
+
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
+ 1. Definitions.
+ 1.1. "Contributor" means each individual or entity that
+ creates or contributes to the creation of Modifications.
+ 1.2. "Contributor Version" means the combination of the
+ Original Software, prior Modifications used by a
+ Contributor (if any), and the Modifications made by that
+ particular Contributor.
+ 1.3. "Covered Software" means (a) the Original Software, or
+ (b) Modifications, or (c) the combination of files
+ containing Original Software with files containing
+ Modifications, in each case including portions thereof.
+ 1.4. "Executable" means the Covered Software in any form
+ other than Source Code.
+ 1.5. "Initial Developer" means the individual or entity
+ that first makes Original Software available under this
+ License.
+ 1.6. "Larger Work" means a work which combines Covered
+ Software or portions thereof with code not governed by the
+ terms of this License.
+ 1.7. "License" means this document.
+ 1.8. "Licensable" means having the right to grant, to the
+ maximum extent possible, whether at the time of the initial
+ grant or subsequently acquired, any and all of the rights
+ conveyed herein.
+ 1.9. "Modifications" means the Source Code and Executable
+ form of any of the following:
+ A. Any file that results from an addition to,
+ deletion from or modification of the contents of a
+ file containing Original Software or previous
+ Modifications;
+ B. Any new file that contains any part of the
+ Original Software or previous Modification; or
+ C. Any new file that is contributed or otherwise made
+ available under the terms of this License.
+ 1.10. "Original Software" means the Source Code and
+ Executable form of computer software code that is
+ originally released under this License.
+ 1.11. "Patent Claims" means any patent claim(s), now owned
+ or hereafter acquired, including without limitation,
+ method, process, and apparatus claims, in any patent
+ Licensable by grantor.
+ 1.12. "Source Code" means (a) the common form of computer
+ software code in which modifications are made and (b)
+ associated documentation included in or with such code.
+ 1.13. "You" (or "Your") means an individual or a legal
+ entity exercising rights under, and complying with all of
+ the terms of, this License. For legal entities, "You"
+ includes any entity which controls, is controlled by, or is
+ under common control with You. For purposes of this
+ definition, "control" means (a) the power, direct or
+ indirect, to cause the direction or management of such
+ entity, whether by contract or otherwise, or (b) ownership
+ of more than fifty percent (50%) of the outstanding shares
+ or beneficial ownership of such entity.
+ 2. License Grants.
+ 2.1. The Initial Developer Grant.
+ Conditioned upon Your compliance with Section 3.1 below and
+ subject to third party intellectual property claims, the
+ Initial Developer hereby grants You a world-wide,
+ royalty-free, non-exclusive license:
+ (a) under intellectual property rights (other than
+ patent or trademark) Licensable by Initial Developer,
+ to use, reproduce, modify, display, perform,
+ sublicense and distribute the Original Software (or
+ portions thereof), with or without Modifications,
+ and/or as part of a Larger Work; and
+ (b) under Patent Claims infringed by the making,
+ using or selling of Original Software, to make, have
+ made, use, practice, sell, and offer for sale, and/or
+ otherwise dispose of the Original Software (or
+ portions thereof).
+ (c) The licenses granted in Sections 2.1(a) and (b)
+ are effective on the date Initial Developer first
+ distributes or otherwise makes the Original Software
+ available to a third party under the terms of this
+ License.
+ (d) Notwithstanding Section 2.1(b) above, no patent
+ license is granted: (1) for code that You delete from
+ the Original Software, or (2) for infringements
+ caused by: (i) the modification of the Original
+ Software, or (ii) the combination of the Original
+ Software with other software or devices.
+ 2.2. Contributor Grant.
+ Conditioned upon Your compliance with Section 3.1 below and
+ subject to third party intellectual property claims, each
+ Contributor hereby grants You a world-wide, royalty-free,
+ non-exclusive license:
+ (a) under intellectual property rights (other than
+ patent or trademark) Licensable by Contributor to
+ use, reproduce, modify, display, perform, sublicense
+ and distribute the Modifications created by such
+ Contributor (or portions thereof), either on an
+ unmodified basis, with other Modifications, as
+ Covered Software and/or as part of a Larger Work; and
+ (b) under Patent Claims infringed by the making,
+ using, or selling of Modifications made by that
+ Contributor either alone and/or in combination with
+ its Contributor Version (or portions of such
+ combination), to make, use, sell, offer for sale,
+ have made, and/or otherwise dispose of: (1)
+ Modifications made by that Contributor (or portions
+ thereof); and (2) the combination of Modifications
+ made by that Contributor with its Contributor Version
+ (or portions of such combination).
+ (c) The licenses granted in Sections 2.2(a) and
+ 2.2(b) are effective on the date Contributor first
+ distributes or otherwise makes the Modifications
+ available to a third party.
+ (d) Notwithstanding Section 2.2(b) above, no patent
+ license is granted: (1) for any code that Contributor
+ has deleted from the Contributor Version; (2) for
+ infringements caused by: (i) third party
+ modifications of Contributor Version, or (ii) the
+ combination of Modifications made by that Contributor
+ with other software (except as part of the
+ Contributor Version) or other devices; or (3) under
+ Patent Claims infringed by Covered Software in the
+ absence of Modifications made by that Contributor.
+ 3. Distribution Obligations.
+ 3.1. Availability of Source Code.
+ Any Covered Software that You distribute or otherwise make
+ available in Executable form must also be made available in
+ Source Code form and that Source Code form must be
+ distributed only under the terms of this License. You must
+ include a copy of this License with every copy of the
+ Source Code form of the Covered Software You distribute or
+ otherwise make available. You must inform recipients of any
+ such Covered Software in Executable form as to how they can
+ obtain such Covered Software in Source Code form in a
+ reasonable manner on or through a medium customarily used
+ for software exchange.
+ 3.2. Modifications.
+ The Modifications that You create or to which You
+ contribute are governed by the terms of this License. You
+ represent that You believe Your Modifications are Your
+ original creation(s) and/or You have sufficient rights to
+ grant the rights conveyed by this License.
+ 3.3. Required Notices.
+ You must include a notice in each of Your Modifications
+ that identifies You as the Contributor of the Modification.
+ You may not remove or alter any copyright, patent or
+ trademark notices contained within the Covered Software, or
+ any notices of licensing or any descriptive text giving
+ attribution to any Contributor or the Initial Developer.
+ 3.4. Application of Additional Terms.
+ You may not offer or impose any terms on any Covered
+ Software in Source Code form that alters or restricts the
+ applicable version of this License or the recipients'
+ rights hereunder. You may choose to offer, and to charge a
+ fee for, warranty, support, indemnity or liability
+ obligations to one or more recipients of Covered Software.
+ However, you may do so only on Your own behalf, and not on
+ behalf of the Initial Developer or any Contributor. You
+ must make it absolutely clear that any such warranty,
+ support, indemnity or liability obligation is offered by
+ You alone, and You hereby agree to indemnify the Initial
+ Developer and every Contributor for any liability incurred
+ by the Initial Developer or such Contributor as a result of
+ warranty, support, indemnity or liability terms You offer.
+ 3.5. Distribution of Executable Versions.
+ You may distribute the Executable form of the Covered
+ Software under the terms of this License or under the terms
+ of a license of Your choice, which may contain terms
+ different from this License, provided that You are in
+ compliance with the terms of this License and that the
+ license for the Executable form does not attempt to limit
+ or alter the recipient's rights in the Source Code form
+ from the rights set forth in this License. If You
+ distribute the Covered Software in Executable form under a
+ different license, You must make it absolutely clear that
+ any terms which differ from this License are offered by You
+ alone, not by the Initial Developer or Contributor. You
+ hereby agree to indemnify the Initial Developer and every
+ Contributor for any liability incurred by the Initial
+ Developer or such Contributor as a result of any such terms
+ You offer.
+ 3.6. Larger Works.
+ You may create a Larger Work by combining Covered Software
+ with other code not governed by the terms of this License
+ and distribute the Larger Work as a single product. In such
+ a case, You must make sure the requirements of this License
+ are fulfilled for the Covered Software.
+ 4. Versions of the License.
+ 4.1. New Versions.
+ Sun Microsystems, Inc. is the initial license steward and
+ may publish revised and/or new versions of this License
+ from time to time. Each version will be given a
+ distinguishing version number. Except as provided in
+ Section 4.3, no one other than the license steward has the
+ right to modify this License.
+ 4.2. Effect of New Versions.
+ You may always continue to use, distribute or otherwise
+ make the Covered Software available under the terms of the
+ version of the License under which You originally received
+ the Covered Software. If the Initial Developer includes a
+ notice in the Original Software prohibiting it from being
+ distributed or otherwise made available under any
+ subsequent version of the License, You must distribute and
+ make the Covered Software available under the terms of the
+ version of the License under which You originally received
+ the Covered Software. Otherwise, You may also choose to
+ use, distribute or otherwise make the Covered Software
+ available under the terms of any subsequent version of the
+ License published by the license steward.
+ 4.3. Modified Versions.
+ When You are an Initial Developer and You want to create a
+ new license for Your Original Software, You may create and
+ use a modified version of this License if You: (a) rename
+ the license and remove any references to the name of the
+ license steward (except to note that the license differs
+ from this License); and (b) otherwise make it clear that
+ the license contains terms which differ from this License.
+ 5. DISCLAIMER OF WARRANTY.
+ COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS"
+ BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
+ INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED
+ SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR
+ PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND
+ PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY
+ COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE
+ INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF
+ ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF
+ WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
+ ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS
+ DISCLAIMER.
+ 6. TERMINATION.
+ 6.1. This License and the rights granted hereunder will
+ terminate automatically if You fail to comply with terms
+ herein and fail to cure such breach within 30 days of
+ becoming aware of the breach. Provisions which, by their
+ nature, must remain in effect beyond the termination of
+ this License shall survive.
+ 6.2. If You assert a patent infringement claim (excluding
+ declaratory judgment actions) against Initial Developer or
+ a Contributor (the Initial Developer or Contributor against
+ whom You assert such claim is referred to as "Participant")
+ alleging that the Participant Software (meaning the
+ Contributor Version where the Participant is a Contributor
+ or the Original Software where the Participant is the
+ Initial Developer) directly or indirectly infringes any
+ patent, then any and all rights granted directly or
+ indirectly to You by such Participant, the Initial
+ Developer (if the Initial Developer is not the Participant)
+ and all Contributors under Sections 2.1 and/or 2.2 of this
+ License shall, upon 60 days notice from Participant
+ terminate prospectively and automatically at the expiration
+ of such 60 day notice period, unless if within such 60 day
+ period You withdraw Your claim with respect to the
+ Participant Software against such Participant either
+ unilaterally or pursuant to a written agreement with
+ Participant.
+ 6.3. In the event of termination under Sections 6.1 or 6.2
+ above, all end user licenses that have been validly granted
+ by You or any distributor hereunder prior to termination
+ (excluding licenses granted to You by any distributor)
+ shall survive termination.
+ 7. LIMITATION OF LIABILITY.
+ UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
+ (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE
+ INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF
+ COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE
+ LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR
+ CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT
+ LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK
+ STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
+ COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
+ INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
+ LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL
+ INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT
+ APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO
+ NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR
+ CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT
+ APPLY TO YOU.
+ 8. U.S. GOVERNMENT END USERS.
+ The Covered Software is a "commercial item," as that term is
+ defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial
+ computer software" (as that term is defined at 48 C.F.R. �
+ 252.227-7014(a)(1)) and "commercial computer software
+ documentation" as such terms are used in 48 C.F.R. 12.212 (Sept.
+ 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1
+ through 227.7202-4 (June 1995), all U.S. Government End Users
+ acquire Covered Software with only those rights set forth herein.
+ This U.S. Government Rights clause is in lieu of, and supersedes,
+ any other FAR, DFAR, or other clause or provision that addresses
+ Government rights in computer software under this License.
+ 9. MISCELLANEOUS.
+ This License represents the complete agreement concerning subject
+ matter hereof. If any provision of this License is held to be
+ unenforceable, such provision shall be reformed only to the
+ extent necessary to make it enforceable. This License shall be
+ governed by the law of the jurisdiction specified in a notice
+ contained within the Original Software (except to the extent
+ applicable law, if any, provides otherwise), excluding such
+ jurisdiction's conflict-of-law provisions. Any litigation
+ relating to this License shall be subject to the jurisdiction of
+ the courts located in the jurisdiction and venue specified in a
+ notice contained within the Original Software, with the losing
+ party responsible for costs, including, without limitation, court
+ costs and reasonable attorneys' fees and expenses. The
+ application of the United Nations Convention on Contracts for the
+ International Sale of Goods is expressly excluded. Any law or
+ regulation which provides that the language of a contract shall
+ be construed against the drafter shall not apply to this License.
+ You agree that You alone are responsible for compliance with the
+ United States export administration regulations (and the export
+ control laws and regulation of any other countries) when You use,
+ distribute or otherwise make available any Covered Software.
+ 10. RESPONSIBILITY FOR CLAIMS.
+ As between Initial Developer and the Contributors, each party is
+ responsible for claims and damages arising, directly or
+ indirectly, out of its utilization of rights under this License
+ and You agree to work with Initial Developer and Contributors to
+ distribute such responsibility on an equitable basis. Nothing
+ herein is intended or shall be deemed to constitute any admission
+ of liability.
+
+------------------------------------------------------------------------------------
+ MIT
+------------------------------------------------------------------------------------
+
+This product bundles slf4j-api 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org
+This product bundles jcl-over-slf4j 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org
+This product bundles jul-to-slf4j 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org
+This product bundles lodash, which is available under a "MIT Software License" license. For details, see https://lodash.com/.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE b/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000..aba445d
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,6 @@
+
+metron-rest
+Copyright 2006-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
new file mode 100644
index 0000000..859ca5e
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker:
+ host:
+ address: 192.168.99.100
+ compose:
+ path: ../../metron-docker/compose/docker-compose.yml
+
+spring:
+ datasource:
+ driverClassName: com.mysql.jdbc.Driver
+ url: jdbc:mysql://${docker.host.address}:3306/metronrest
+ username: root
+ password: root
+ platform: mysql
+ jpa:
+ hibernate:
+ ddl-auto: update
+
+zookeeper:
+ url: ${docker.host.address}:2181
+
+kafka:
+ broker:
+ url: ${docker.host.address}:9092
+
+hdfs:
+ namenode:
+ url: file:///
+
+grok:
+ path:
+ temp: target/patterns/temp
+ default: target/patterns
+
+storm:
+ ui:
+ url: ${docker.host.address}:8080
+ parser:
+ script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh
+ enrichment:
+ script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
+ indexing:
+ script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/application-test.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml
new file mode 100644
index 0000000..2872e95
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+spring:
+ datasource:
+ driverClassName: org.h2.Driver
+ url: jdbc:h2:mem:testdb
+ username: sa
+ password:
+ platform: h2
+ jpa:
+ hibernate:
+ ddl-auto: create-drop
+ main:
+ banner-mode: off
+
+grok:
+ path:
+ temp: target/patterns/temp
+ default: target/patterns
+
+storm:
+ ui:
+ url: stormUITestUrl
+ parser:
+ script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh
+ enrichment:
+ script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
+ indexing:
+ script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
new file mode 100644
index 0000000..7e45ed9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+metron:
+ version: 0.3.0
+
+logging:
+ level:
+ root: ERROR
+
+spring:
+ jackson:
+ date-format: yyyy-MM-dd HH:mm:ss
+
+grok:
+ path:
+ temp: ./
+ default: /apps/metron/patterns
+
+zookeeper:
+ client:
+ timeout:
+ session: 10000
+ connection: 10000
+
+curator:
+ sleep:
+ time: 1000
+ max:
+ retries: 3
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/log4j.properties b/metron-interface/metron-rest/src/main/resources/log4j.properties
new file mode 100644
index 0000000..492cecf
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/log4j.properties
@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=ERROR, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/schema-h2.sql
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/schema-h2.sql b/metron-interface/metron-rest/src/main/resources/schema-h2.sql
new file mode 100644
index 0000000..986b652
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/schema-h2.sql
@@ -0,0 +1,30 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+drop table if exists authorities;
+drop table if exists users;
+
+create table if not exists users(
+ username varchar(50) not null primary key,
+ password varchar(50) not null,
+ enabled boolean not null
+);
+create table authorities (
+ username varchar(50) not null,
+ authority varchar(50) not null,
+ constraint fk_authorities_users foreign key(username) references users(username)
+);
+create unique index ix_auth_username on authorities (username,authority);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/schema-mysql.sql b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
new file mode 100644
index 0000000..986b652
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
@@ -0,0 +1,30 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+drop table if exists authorities;
+drop table if exists users;
+
+create table if not exists users(
+ username varchar(50) not null primary key,
+ password varchar(50) not null,
+ enabled boolean not null
+);
+create table authorities (
+ username varchar(50) not null,
+ authority varchar(50) not null,
+ constraint fk_authorities_users foreign key(username) references users(username)
+);
+create unique index ix_auth_username on authorities (username,authority);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/main/scripts/start.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start.sh b/metron-interface/metron-rest/src/main/scripts/start.sh
new file mode 100755
index 0000000..d83a2fd
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/scripts/start.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=0.3.0
+SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+if [ "$#" -ne 1 ]; then
+ echo "Usage: start.sh <path to application.yml file>"
+ echo "Path can be absolute or relative to $SCRIPTS_ROOT"
+else
+ if [ -z "$MYSQL_CLIENT_HOME" ]; then
+ echo MYSQL_CLIENT_HOME is not set
+ else
+ if [ -z "$HIBERNATE_HOME" ]; then
+ echo HIBERNATE_HOME is not set
+ else
+ java -Dloader.path=$MYSQL_CLIENT_HOME/*,$HIBERNATE_HOME/lib/envers/*,$HIBERNATE_HOME/lib/jpa/*,$HIBERNATE_HOME/lib/required/* -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$1
+ fi
+ fi
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java
new file mode 100644
index 0000000..3025932
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.boot.SpringApplication;
+
+import static org.mockito.Mockito.times;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.verifyStatic;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({SpringApplication.class})
+public class MetronRestApplicationTest {
+
+ @Test
+ public void test() {
+ mockStatic(SpringApplication.class);
+ String[] args = {"arg1", "arg2"};
+ MetronRestApplication.main(args);
+ verifyStatic(times(1));
+ SpringApplication.run(MetronRestApplication.class, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
new file mode 100644
index 0000000..7931fe6
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import com.google.common.base.Function;
+import kafka.utils.ZKStringSerializer$;
+import kafka.utils.ZkUtils;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.integration.components.KafkaComponent;
+import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.rest.mock.MockStormCLIClientWrapper;
+import org.apache.metron.rest.mock.MockStormRestTemplate;
+import org.apache.metron.rest.service.impl.StormCLIWrapper;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.web.client.RestTemplate;
+
+import javax.annotation.Nullable;
+import java.util.Properties;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile(TEST_PROFILE)
+public class TestConfig {
+
+ @Bean
+ public Properties zkProperties() {
+ return new Properties();
+ }
+
+ @Bean
+ public ZKServerComponent zkServerComponent(Properties zkProperties) {
+ return new ZKServerComponent()
+ .withPostStartCallback((zkComponent) -> zkProperties.setProperty(ZKServerComponent.ZOOKEEPER_PROPERTY, zkComponent.getConnectionString()));
+ }
+
+ @Bean
+ public KafkaComponent kafkaWithZKComponent(Properties zkProperties) {
+ return new KafkaComponent().withTopologyProperties(zkProperties);
+ }
+
+ //@Bean(destroyMethod = "stop")
+ @Bean
+ public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) {
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("zk", zkServerComponent)
+ .withComponent("kafka", kafkaWithZKComponent)
+ .withCustomShutdownOrder(new String[] {"kafka","zk"})
+ .build();
+ try {
+ runner.start();
+ } catch (UnableToStartException e) {
+ e.printStackTrace();
+ }
+ return runner;
+ }
+
+ @Bean(initMethod = "start", destroyMethod="close")
+ public CuratorFramework client(ComponentRunner componentRunner) {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ ZKServerComponent zkServerComponent = componentRunner.getComponent("zk", ZKServerComponent.class);
+ return CuratorFrameworkFactory.newClient(zkServerComponent.getConnectionString(), retryPolicy);
+ }
+
+ @Bean(destroyMethod="close")
+ public ZkClient zkClient(ComponentRunner componentRunner) {
+ ZKServerComponent zkServerComponent = componentRunner.getComponent("zk", ZKServerComponent.class);
+ return new ZkClient(zkServerComponent.getConnectionString(), 10000, 10000, ZKStringSerializer$.MODULE$);
+ }
+
+ @Bean
+ public ZkUtils zkUtils(ZkClient zkClient) {
+ return ZkUtils.apply(zkClient, false);
+ }
+
+ @Bean(destroyMethod="close")
+ public KafkaConsumer<String, String> kafkaConsumer(KafkaComponent kafkaWithZKComponent) {
+ Properties props = new Properties();
+ props.put("bootstrap.servers", kafkaWithZKComponent.getBrokerList());
+ props.put("group.id", "metron-config");
+ props.put("enable.auto.commit", "false");
+ props.put("auto.commit.interval.ms", "1000");
+ props.put("session.timeout.ms", "30000");
+ props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+ return new KafkaConsumer<>(props);
+ }
+
+ @Bean
+ public StormCLIWrapper stormCLIClientWrapper() {
+ return new MockStormCLIClientWrapper();
+ }
+
+ @Bean
+ public RestTemplate restTemplate(StormCLIWrapper stormCLIClientWrapper) {
+ MockStormRestTemplate restTemplate = new MockStormRestTemplate();
+ restTemplate.setMockStormCLIClientWrapper((MockStormCLIClientWrapper) stormCLIClientWrapper);
+ return restTemplate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
new file mode 100644
index 0000000..7796c8b
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class GlobalConfigControllerIntegrationTest {
+
+ /**
+ {
+ "solr.zookeeper": "solr:2181",
+ "solr.collection": "metron",
+ "solr.numShards": 1,
+ "solr.replicationFactor": 1
+ }
+ */
+ @Multiline
+ public static String globalJson;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String globalConfigUrl = "/api/v1/global/config";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(globalConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(globalConfigUrl))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(delete(globalConfigUrl).with(csrf()))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+
+ this.mockMvc.perform(post(globalConfigUrl).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson))
+ .andExpect(status().isCreated())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")));
+
+ this.mockMvc.perform(post(globalConfigUrl).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")));
+
+ this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password)))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isOk());
+
+ this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf()))
+ .andExpect(status().isNotFound());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/55b3e7ea/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
new file mode 100644
index 0000000..e618d48
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class GrokControllerIntegrationTest {
+
+ /**
+ {
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+ "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}"
+ }
+ */
+ @Multiline
+ public static String grokValidationJson;
+
+ /**
+ {
+ "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html",
+ "statement":""
+ }
+ */
+ @Multiline
+ public static String badGrokValidationJson;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String grokUrl = "/api/v1/grok";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(grokUrl + "/validate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(grokValidationJson))
+ .andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(get(grokUrl + "/list"))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(grokValidationJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.results.action").value("TCP_MISS"))
+ .andExpect(jsonPath("$.results.bytes").value(337891))
+ .andExpect(jsonPath("$.results.code").value(200))
+ .andExpect(jsonPath("$.results.elapsed").value(415))
+ .andExpect(jsonPath("$.results.ip_dst_addr").value("207.109.73.154"))
+ .andExpect(jsonPath("$.results.ip_src_addr").value("127.0.0.1"))
+ .andExpect(jsonPath("$.results.method").value("GET"))
+ .andExpect(jsonPath("$.results.timestamp").value("1467011157.401"))
+ .andExpect(jsonPath("$.results.url").value("http://www.aliexpress.com/af/shoes.html?"));
+
+ this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(badGrokValidationJson))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.responseCode").value(500))
+ .andExpect(jsonPath("$.message").value("A pattern label must be included (ex. PATTERN_LABEL ${PATTERN:field} ...)"));
+
+ this.mockMvc.perform(get(grokUrl + "/list").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$").isNotEmpty());
+ }
+}
[08/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
new file mode 100644
index 0000000..d292948
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.GetChildrenBuilder;
+import org.apache.curator.framework.api.GetDataBuilder;
+import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorEnrichmentConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class SensorEnrichmentConfigServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ ObjectMapper objectMapper;
+ CuratorFramework curatorFramework;
+ SensorEnrichmentConfigService sensorEnrichmentConfigService;
+
+ /**
+ {
+ "enrichment" : {
+ "fieldMap": {
+ "geo": ["ip_dst_addr"]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": ["ip_src_addr"]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr" : ["malicious_ip"]
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ @Before
+ public void setUp() throws Exception {
+ objectMapper = mock(ObjectMapper.class);
+ curatorFramework = mock(CuratorFramework.class);
+ sensorEnrichmentConfigService = new SensorEnrichmentConfigServiceImpl(objectMapper, curatorFramework);
+ }
+
+
+ @Test
+ public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ assertFalse(sensorEnrichmentConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ assertFalse(sensorEnrichmentConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(null);
+
+ assertTrue(sensorEnrichmentConfigService.delete("bro"));
+
+ verify(curatorFramework).delete();
+ }
+
+ @Test
+ public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception {
+ final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig();
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(getTestSensorEnrichmentConfig(), sensorEnrichmentConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertNull(sensorEnrichmentConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ sensorEnrichmentConfigService.findOne("bro");
+ }
+
+ @Test
+ public void getAllTypesShouldProperlyReturnTypes() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }}, sensorEnrichmentConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList<>(), sensorEnrichmentConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())).thenThrow(Exception.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ sensorEnrichmentConfigService.getAllTypes();
+ }
+
+ @Test
+ public void getAllShouldProperlyReturnSensorEnrichmentConfigs() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig();
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(new HashMap() {{ put("bro", sensorEnrichmentConfig);}}, sensorEnrichmentConfigService.getAll());
+ }
+
+ @Test
+ public void saveShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ sensorEnrichmentConfigService.save("bro", new SensorEnrichmentConfig());
+ }
+
+ @Test
+ public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
+ final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig();
+
+ when(objectMapper.writeValueAsString(sensorEnrichmentConfig)).thenReturn(broJson);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ assertEquals(sensorEnrichmentConfig, sensorEnrichmentConfigService.save("bro", sensorEnrichmentConfig));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ }
+
+ @Test
+ public void getAvailableEnrichmentsShouldReturnEnrichments() throws Exception {
+ assertEquals(new ArrayList<String>() {{
+ add("geo");
+ add("host");
+ add("whois");
+ }}, sensorEnrichmentConfigService.getAvailableEnrichments());
+ }
+
+ private SensorEnrichmentConfig getTestSensorEnrichmentConfig() {
+ SensorEnrichmentConfig sensorEnrichmentConfig = new SensorEnrichmentConfig();
+ EnrichmentConfig enrichmentConfig = new EnrichmentConfig();
+ enrichmentConfig.setFieldMap(new HashMap() {{ put("geo", Arrays.asList("ip_dst_addr")); }});
+ sensorEnrichmentConfig.setEnrichment(enrichmentConfig);
+ ThreatIntelConfig threatIntelConfig = new ThreatIntelConfig();
+ threatIntelConfig.setFieldMap(new HashMap() {{ put("hbaseThreatIntel", Arrays.asList("ip_src_addr")); }});
+ threatIntelConfig.setFieldToTypeMap(new HashMap() {{ put("ip_src_addr", Arrays.asList("malicious_ip")); }});
+ sensorEnrichmentConfig.setThreatIntel(threatIntelConfig);
+ return sensorEnrichmentConfig;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
new file mode 100644
index 0000000..43ca0f7
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.GetChildrenBuilder;
+import org.apache.curator.framework.api.GetDataBuilder;
+import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.SensorIndexingConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class SensorIndexingConfigServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ ObjectMapper objectMapper;
+ CuratorFramework curatorFramework;
+ SensorIndexingConfigService sensorIndexingConfigService;
+
+ /**
+ {
+ "hdfs" : {
+ "index": "bro",
+ "batchSize": 5,
+ "enabled" : true
+ }
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ @Before
+ public void setUp() throws Exception {
+ objectMapper = mock(ObjectMapper.class);
+ curatorFramework = mock(CuratorFramework.class);
+ sensorIndexingConfigService = new SensorIndexingConfigServiceImpl(objectMapper, curatorFramework);
+ }
+
+
+ @Test
+ public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ assertFalse(sensorIndexingConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ assertFalse(sensorIndexingConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(null);
+
+ assertTrue(sensorIndexingConfigService.delete("bro"));
+
+ verify(curatorFramework).delete();
+ }
+
+ @Test
+ public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception {
+ final Map<String, Object> sensorIndexingConfig = getTestSensorIndexingConfig();
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(getTestSensorIndexingConfig(), sensorIndexingConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertNull(sensorIndexingConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ sensorIndexingConfigService.findOne("bro");
+ }
+
+ @Test
+ public void getAllTypesShouldProperlyReturnTypes() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }}, sensorIndexingConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList<>(), sensorIndexingConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())).thenThrow(Exception.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ sensorIndexingConfigService.getAllTypes();
+ }
+
+ @Test
+ public void getAllShouldProperlyReturnSensorEnrichmentConfigs() throws Exception {
+ final Map<String, Object> sensorIndexingConfig = getTestSensorIndexingConfig();
+
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(new HashMap() {{ put("bro", sensorIndexingConfig);}}, sensorIndexingConfigService.getAll());
+ }
+
+ @Test
+ public void saveShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ sensorIndexingConfigService.save("bro", new HashMap<>());
+ }
+
+ @Test
+ public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
+ final Map<String, Object> sensorIndexingConfig = getTestSensorIndexingConfig();
+
+ when(objectMapper.writeValueAsString(sensorIndexingConfig)).thenReturn(broJson);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ assertEquals(sensorIndexingConfig, sensorIndexingConfigService.save("bro", sensorIndexingConfig));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ }
+
+ private Map<String, Object> getTestSensorIndexingConfig() {
+ Map<String, Object> sensorIndexingConfig = new HashMap<>();
+ sensorIndexingConfig.put("hdfs", new HashMap(){{
+ put("index", "bro");
+ put("batchSize", 5);
+ put("enabled", true);
+ }});
+ return sensorIndexingConfig;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
new file mode 100644
index 0000000..d35a48c
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.GetChildrenBuilder;
+import org.apache.curator.framework.api.GetDataBuilder;
+import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.ParseMessageRequest;
+import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.springframework.core.env.Environment;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class SensorParserConfigServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ Environment environment;
+ ObjectMapper objectMapper;
+ CuratorFramework curatorFramework;
+ GrokService grokService;
+ SensorParserConfigService sensorParserConfigService;
+
+ /**
+ {
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squid",
+ "parserConfig": {
+ "grokPath": "/patterns/squid",
+ "patternLabel": "SQUID_DELIMITED",
+ "timestampField": "timestamp"
+ }
+ }
+ */
+ @Multiline
+ public static String squidJson;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic":"bro",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ public static String broJson;
+
+ @Before
+ public void setUp() throws Exception {
+ objectMapper = mock(ObjectMapper.class);
+ curatorFramework = mock(CuratorFramework.class);
+ grokService = mock(GrokService.class);
+ sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService);
+ }
+
+
+ @Test
+ public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ assertFalse(sensorParserConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ assertFalse(sensorParserConfigService.delete("bro"));
+ }
+
+ @Test
+ public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception {
+ DeleteBuilder builder = mock(DeleteBuilder.class);
+
+ when(curatorFramework.delete()).thenReturn(builder);
+ when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(null);
+
+ assertTrue(sensorParserConfigService.delete("bro"));
+
+ verify(curatorFramework).delete();
+ }
+
+ @Test
+ public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception {
+ final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig();
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception {
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertNull(sensorParserConfigService.findOne("bro"));
+ }
+
+ @Test
+ public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class);
+
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ sensorParserConfigService.findOne("bro");
+ }
+
+ @Test
+ public void getAllTypesShouldProperlyReturnTypes() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }}, sensorParserConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldReturnEmptyListWhenNoNodeExceptionIsThrown() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ assertEquals(new ArrayList<>(), sensorParserConfigService.getAllTypes());
+ }
+
+ @Test
+ public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(Exception.class);
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ sensorParserConfigService.getAllTypes();
+ }
+
+ @Test
+ public void getAllShouldProperlyReturnSensorParserConfigs() throws Exception {
+ GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
+ when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot()))
+ .thenReturn(new ArrayList() {{
+ add("bro");
+ add("squid");
+ }});
+ when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
+
+ final SensorParserConfig broSensorParserConfig = getTestBroSensorParserConfig();
+ final SensorParserConfig squidSensorParserConfig = getTestSquidSensorParserConfig();
+ GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
+ when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes());
+ when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/squid")).thenReturn(squidJson.getBytes());
+ when(curatorFramework.getData()).thenReturn(getDataBuilder);
+
+ assertEquals(new ArrayList() {{
+ add(getTestBroSensorParserConfig());
+ add(getTestSquidSensorParserConfig());
+ }}, sensorParserConfigService.getAll());
+ }
+
+ @Test
+ public void saveShouldWrapExceptionInRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class);
+
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ final SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setSensorTopic("bro");
+ sensorParserConfigService.save(sensorParserConfig);
+ }
+
+ @Test
+ public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
+ final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig();
+
+ when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(broJson);
+
+ SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
+ when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat());
+ when(curatorFramework.setData()).thenReturn(setDataBuilder);
+
+ assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save(sensorParserConfig));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes()));
+ }
+
+ @Test
+ public void reloadAvailableParsersShouldReturnParserClasses() throws Exception {
+ Map<String, String> availableParsers = sensorParserConfigService.reloadAvailableParsers();
+ assertTrue(availableParsers.size() > 0);
+ assertEquals("org.apache.metron.parsers.GrokParser", availableParsers.get("Grok"));
+ assertEquals("org.apache.metron.parsers.bro.BasicBroParser", availableParsers.get("Bro"));
+ }
+
+ @Test
+ public void parseMessageShouldProperlyReturnParsedResults() throws Exception {
+ final SensorParserConfig sensorParserConfig = getTestSquidSensorParserConfig();
+ String grokStatement = "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?";
+ String sampleData = "1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html";
+ ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
+ parseMessageRequest.setSensorParserConfig(sensorParserConfig);
+ parseMessageRequest.setGrokStatement(grokStatement);
+ parseMessageRequest.setSampleData(sampleData);
+
+ File patternFile = new File("./target/squidTest");
+ FileWriter writer = new FileWriter(patternFile);
+ writer.write(grokStatement);
+ writer.close();
+
+ when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile);
+
+ assertEquals(new HashMap() {{
+ put("elapsed", 161);
+ put("code", 200);
+ put("ip_dst_addr", "199.27.79.73");
+ put("ip_src_addr", "127.0.0.1");
+ put("action", "TCP_MISS");
+ put("bytes", 103701);
+ put("method", "GET");
+ put("url", "http://www.cnn.com/");
+ put("timestamp", 1461576382642L);
+ put("original_string", "1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html");
+ }}, sensorParserConfigService.parseMessage(parseMessageRequest));
+
+ }
+
+ @Test
+ public void missingSensorParserConfigShouldThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
+ sensorParserConfigService.parseMessage(parseMessageRequest);
+ }
+
+ @Test
+ public void missingParserClassShouldThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ final SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setSensorTopic("squid");
+ ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
+ parseMessageRequest.setSensorParserConfig(sensorParserConfig);
+ sensorParserConfigService.parseMessage(parseMessageRequest);
+ }
+
+ @Test
+ public void invalidParserClassShouldThrowRestException() throws Exception {
+ exception.expect(RestException.class);
+
+ final SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setSensorTopic("squid");
+ sensorParserConfig.setParserClassName("bad.class.package.BadClassName");
+ ParseMessageRequest parseMessageRequest = new ParseMessageRequest();
+ parseMessageRequest.setSensorParserConfig(sensorParserConfig);
+ sensorParserConfigService.parseMessage(parseMessageRequest);
+ }
+
+ private SensorParserConfig getTestBroSensorParserConfig() {
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setSensorTopic("bro");
+ sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser");
+ return sensorParserConfig;
+ }
+
+ private SensorParserConfig getTestSquidSensorParserConfig() {
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setSensorTopic("squid");
+ sensorParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser");
+ sensorParserConfig.setParserConfig(new HashMap() {{
+ put("grokPath", "/patterns/squid");
+ put("patternLabel", "SQUID_DELIMITED");
+ put("timestampField", "timestamp");
+ }});
+ return sensorParserConfig;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java
new file mode 100644
index 0000000..3ddbf56
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.apache.storm.shade.com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class StellarServiceImplTest {
+
+ private StellarService stellarService;
+
+ @Before
+ public void setUp() throws Exception {
+ stellarService = new StellarServiceImpl();
+ }
+
+ @Test
+ public void validateRulesShouldProperlyValidateRules() {
+ List<String> rules = Arrays.asList("TO_LOWER(test)", "BAD_FUNCTION(test)");
+ Map<String, Boolean> results = stellarService.validateRules(rules);
+ assertEquals(2, results.size());
+ assertEquals(true, results.get("TO_LOWER(test)"));
+ assertEquals(false, results.get("BAD_FUNCTION(test)"));
+ }
+
+ @Test
+ public void applyTransformationsShouldProperlyTransformData() {
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ FieldTransformer fieldTransformater = new FieldTransformer();
+ fieldTransformater.setOutput("url_host");
+ fieldTransformater.setTransformation("STELLAR");
+ fieldTransformater.setConfig(new LinkedHashMap<String, Object>() {{
+ put("url_host", "TO_LOWER(URL_TO_HOST(url))");
+ }});
+ sensorParserConfig.setFieldTransformations(ImmutableList.of(fieldTransformater));
+ SensorParserContext sensorParserContext = new SensorParserContext();
+ sensorParserContext.setSensorParserConfig(sensorParserConfig);
+ sensorParserContext.setSampleData(new HashMap<String, Object>() {{
+ put("url", "https://caseystella.com/blog");
+ }});
+ Map<String, Object> results = stellarService.applyTransformations(sensorParserContext);
+ assertEquals(2, results.size());
+ assertEquals("https://caseystella.com/blog", results.get("url"));
+ assertEquals("caseystella.com", results.get("url_host"));
+ }
+
+ @Test
+ public void getTransformationsShouldReturnTransformation() {
+ assertTrue(stellarService.getTransformations().length > 0);
+ }
+
+ @Test
+ public void getStellarFunctionsShouldReturnFunctions() {
+ assertTrue(stellarService.getStellarFunctions().size() > 0);
+ }
+
+ @Test
+ public void getSimpleStellarFunctionsShouldReturnFunctions() {
+ assertEquals(1, stellarService.getSimpleStellarFunctions().stream()
+ .filter(stellarFunctionDescription -> stellarFunctionDescription.getName().equals("TO_LOWER")).count());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
new file mode 100644
index 0000000..d83a74c
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.metron.rest.service.SensorParserConfigService;
+import org.apache.metron.rest.service.StormAdminService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class StormAdminServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ StormCLIWrapper stormCLIClientWrapper;
+ StormAdminService stormAdminService;
+ GlobalConfigService globalConfigService;
+ SensorParserConfigService sensorParserConfigService;
+
+ @Before
+ public void setUp() throws Exception {
+ stormCLIClientWrapper = mock(StormCLIWrapper.class);
+ globalConfigService = mock(GlobalConfigService.class);
+ sensorParserConfigService = mock(SensorParserConfigService.class);
+ stormAdminService = new StormAdminServiceImpl(stormCLIClientWrapper, globalConfigService, sensorParserConfigService);
+ }
+
+ @Test
+ public void startParserTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.startParserTopology("bro")).thenReturn(0);
+ when(globalConfigService.get()).thenReturn(new HashMap<String, Object>());
+ when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig());
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.STARTED.toString());
+ TopologyResponse actual = stormAdminService.startParserTopology("bro");
+
+ assertEquals(expected, actual);
+ assertEquals(expected.hashCode(), actual.hashCode());
+ }
+
+ @Test
+ public void startParserTopologyShouldReturnGlobalConfigMissingError() throws Exception {
+ when(globalConfigService.get()).thenReturn(null);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage(TopologyStatusCode.GLOBAL_CONFIG_MISSING.toString());
+
+ assertEquals(expected, stormAdminService.startParserTopology("bro"));
+ }
+
+ @Test
+ public void startParserTopologyShouldReturnSensorParserConfigMissingError() throws Exception {
+ when(globalConfigService.get()).thenReturn(new HashMap<String, Object>());
+ when(sensorParserConfigService.findOne("bro")).thenReturn(null);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString());
+
+ assertEquals(expected, stormAdminService.startParserTopology("bro"));
+ }
+
+ @Test
+ public void stopParserTopologyShouldProperlyReturnErrorTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.stopParserTopology("bro", false)).thenReturn(1);
+ when(globalConfigService.get()).thenReturn(new HashMap<String, Object>());
+ when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig());
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage(TopologyStatusCode.STOP_ERROR.toString());
+
+ assertEquals(expected, stormAdminService.stopParserTopology("bro", false));
+ }
+
+ @Test
+ public void startEnrichmentTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.startEnrichmentTopology()).thenReturn(0);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.STARTED.toString());
+
+ assertEquals(expected, stormAdminService.startEnrichmentTopology());
+ }
+
+ @Test
+ public void stopEnrichmentTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.stopEnrichmentTopology(false)).thenReturn(0);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.STOPPED.toString());
+
+ assertEquals(expected, stormAdminService.stopEnrichmentTopology(false));
+ }
+
+ @Test
+ public void startIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.startIndexingTopology()).thenReturn(0);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.STARTED.toString());
+
+ assertEquals(expected, stormAdminService.startIndexingTopology());
+ }
+
+ @Test
+ public void stopIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
+ when(stormCLIClientWrapper.stopIndexingTopology(false)).thenReturn(0);
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.STOPPED.toString());
+
+ assertEquals(expected, stormAdminService.stopIndexingTopology(false));
+ }
+
+ @Test
+ public void getStormClientStatusShouldProperlyReturnStatus() throws Exception {
+ final Map<String, String> status = new HashMap() {{
+ put("status", "statusValue");
+ }};
+ when(stormCLIClientWrapper.getStormClientStatus()).thenReturn(status);
+
+ assertEquals(new HashMap() {{
+ put("status", "statusValue");
+ }}, stormAdminService.getStormClientStatus());
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
new file mode 100644
index 0000000..cb26783
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.core.env.Environment;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.mock;
+import static org.powermock.api.mockito.PowerMockito.verifyNew;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.powermock.api.mockito.PowerMockito.whenNew;
+
+@SuppressWarnings("unchecked")
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DockerStormCLIWrapper.class, ProcessBuilder.class})
+public class StormCLIWrapperTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private ProcessBuilder processBuilder;
+ private Environment environment;
+ private Process process;
+ private StormCLIWrapper stormCLIWrapper;
+
+ @Before
+ public void setUp() throws Exception {
+ processBuilder = mock(ProcessBuilder.class);
+ environment = mock(Environment.class);
+ process = mock(Process.class);
+ stormCLIWrapper = new StormCLIWrapper();
+ stormCLIWrapper.setEnvironment(environment);
+ }
+
+ @Test
+ public void startParserTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_parser");
+ when(environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY)).thenReturn("kafka_broker_url");
+ when(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY)).thenReturn("zookeeper_url");
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.startParserTopology("bro"));
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("/start_parser", "-k", "kafka_broker_url", "-z", "zookeeper_url", "-s", "bro");
+ }
+
+ @Test
+ public void stopParserTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.stopParserTopology("bro", false));
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("storm", "kill", "bro");
+ }
+
+ @Test
+ public void stopParserTopologyNowShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.stopParserTopology("bro", true));
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("storm", "kill", "bro", "-w", "0");
+ }
+
+ @Test
+ public void startEnrichmentTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_enrichment");
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.startEnrichmentTopology());
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("/start_enrichment");
+
+ }
+
+ @Test
+ public void stopEnrichmentTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.stopEnrichmentTopology(false));
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME);
+ }
+
+ @Test
+ public void startIndexingTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing");
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.startIndexingTopology());
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("/start_indexing");
+
+ }
+
+ @Test
+ public void stopIndexingTopologyShouldRunCommandProperly() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.exitValue()).thenReturn(0);
+
+ assertEquals(0, stormCLIWrapper.stopIndexingTopology(false));
+ verify(process).waitFor();
+ verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.INDEXING_TOPOLOGY_NAME);
+ }
+
+ @Test
+ public void getStormClientStatusShouldReturnCorrectStatus() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ Process process = mock(Process.class);
+ InputStream inputStream = new ByteArrayInputStream("\nStorm 1.1".getBytes(UTF_8));
+
+ when(processBuilder.start()).thenReturn(process);
+
+ when(process.getInputStream()).thenReturn(inputStream);
+ when(environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_parser");
+ when(environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_enrichment");
+ when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing");
+
+
+ Map<String, String> actual = stormCLIWrapper.getStormClientStatus();
+ assertEquals(new HashMap<String, String>() {{
+ put("parserScriptPath", "/start_parser");
+ put("enrichmentScriptPath", "/start_enrichment");
+ put("indexingScriptPath", "/start_indexing");
+ put("stormClientVersionInstalled", "1.1");
+
+ }}, actual);
+ verifyNew(ProcessBuilder.class).withArguments("storm", "version");
+ }
+
+ @Test
+ public void stormClientVersionInstalledShouldReturnDefault() throws Exception {
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+
+ Process process = mock(Process.class);
+ InputStream inputStream = new ByteArrayInputStream("".getBytes(UTF_8));
+
+ when(processBuilder.start()).thenReturn(process);
+ when(process.getInputStream()).thenReturn(inputStream);
+ assertEquals("Storm client is not installed", stormCLIWrapper.stormClientVersionInstalled());
+ }
+
+ @Test
+ public void runCommandShouldReturnRestExceptionOnError() throws Exception {
+ exception.expect(RestException.class);
+
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+ when(processBuilder.start()).thenThrow(new IOException());
+
+ stormCLIWrapper.runCommand(new String[]{"storm", "kill"});
+ }
+
+ @Test
+ public void stormClientVersionInstalledShouldReturnRestExceptionOnError() throws Exception {
+ exception.expect(RestException.class);
+
+ whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
+ when(processBuilder.start()).thenThrow(new IOException());
+
+ stormCLIWrapper.stormClientVersionInstalled();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java
new file mode 100644
index 0000000..db2cb7f
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.rest.model.TopologyResponse;
+import org.apache.metron.rest.model.TopologyStatus;
+import org.apache.metron.rest.model.TopologyStatusCode;
+import org.apache.metron.rest.model.TopologySummary;
+import org.apache.metron.rest.service.StormStatusService;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.springframework.core.env.Environment;
+import org.springframework.web.client.RestTemplate;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.metron.rest.MetronRestConstants.STORM_UI_SPRING_PROPERTY;
+import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_SUMMARY_URL;
+import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class StormStatusServiceImplTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ Environment environment;
+ RestTemplate restTemplate;
+ StormStatusService stormStatusService;
+
+ @Before
+ public void setUp() throws Exception {
+ environment = mock(Environment.class);
+ restTemplate = mock(RestTemplate.class);
+ stormStatusService = new StormStatusServiceImpl(environment, restTemplate);
+ }
+
+ @Test
+ public void getTopologySummaryShouldReturnTopologySummary() throws Exception {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setStatus(TopologyStatusCode.STARTED);
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+
+ TopologyStatus expectedStatus = new TopologyStatus();
+ expectedStatus.setStatus(TopologyStatusCode.STARTED);
+ expectedStatus.setName("bro");
+ expectedStatus.setId("bro_id");
+ TopologySummary expected = new TopologySummary();
+ expected.setTopologies(new TopologyStatus[]{expectedStatus});
+
+ TopologySummary actual = stormStatusService.getTopologySummary();
+ assertEquals(expected, actual);
+ assertEquals(expected.hashCode(), actual.hashCode());
+ }
+
+ @Test
+ public void getTopologyStatusShouldReturnTopologyStatus() throws Exception {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setStatus(TopologyStatusCode.STARTED);
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id", TopologyStatus.class)).thenReturn(topologyStatus);
+
+ TopologyStatus expected = new TopologyStatus();
+ expected.setStatus(TopologyStatusCode.STARTED);
+ expected.setName("bro");
+ expected.setId("bro_id");
+
+ TopologyStatus actual = stormStatusService.getTopologyStatus("bro");
+ assertEquals(expected, actual);
+ assertEquals(expected.hashCode(), actual.hashCode());
+ }
+
+ @Test
+ public void getAllTopologyStatusShouldReturnAllTopologyStatus() {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setStatus(TopologyStatusCode.STARTED);
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id", TopologyStatus.class)).thenReturn(topologyStatus);
+
+ TopologyStatus expected = new TopologyStatus();
+ expected.setStatus(TopologyStatusCode.STARTED);
+ expected.setName("bro");
+ expected.setId("bro_id");
+
+ assertEquals(new ArrayList() {{ add(expected); }}, stormStatusService.getAllTopologyStatus());
+ }
+
+
+ @Test
+ public void activateTopologyShouldReturnActiveTopologyResponse() {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/activate", null, Map.class))
+ .thenReturn(new HashMap() {{ put("status", "success"); }});
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.ACTIVE.toString());
+ assertEquals(expected, stormStatusService.activateTopology("bro"));
+ }
+
+ @Test
+ public void activateTopologyShouldReturnErrorTopologyResponse() {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/activate", null, Map.class))
+ .thenReturn(new HashMap() {{ put("status", "error message"); }});
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage("error message");
+ assertEquals(expected, stormStatusService.activateTopology("bro"));
+ }
+
+ @Test
+ public void activateTopologyShouldReturnTopologyNotFoundTopologyResponse() {
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(new TopologySummary());
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString());
+ assertEquals(expected, stormStatusService.activateTopology("bro"));
+ }
+
+ @Test
+ public void deactivateTopologyShouldReturnActiveTopologyResponse() {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/deactivate", null, Map.class))
+ .thenReturn(new HashMap() {{ put("status", "success"); }});
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setSuccessMessage(TopologyStatusCode.INACTIVE.toString());
+ assertEquals(expected, stormStatusService.deactivateTopology("bro"));
+ }
+
+ @Test
+ public void deactivateTopologyShouldReturnErrorTopologyResponse() {
+ final TopologyStatus topologyStatus = new TopologyStatus();
+ topologyStatus.setName("bro");
+ topologyStatus.setId("bro_id");
+ final TopologySummary topologySummary = new TopologySummary();
+ topologySummary.setTopologies(new TopologyStatus[]{topologyStatus});
+
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary);
+ when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/deactivate", null, Map.class))
+ .thenReturn(new HashMap() {{ put("status", "error message"); }});
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage("error message");
+ assertEquals(expected, stormStatusService.deactivateTopology("bro"));
+ }
+
+ @Test
+ public void deactivateTopologyShouldReturnTopologyNotFoundTopologyResponse() {
+ when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui");
+ when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(new TopologySummary());
+
+ TopologyResponse expected = new TopologyResponse();
+ expected.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString());
+ assertEquals(expected, stormStatusService.deactivateTopology("bro"));
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/test/resources/README.vm
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/resources/README.vm b/metron-interface/metron-rest/src/test/resources/README.vm
index f259a91..99f4286 100644
--- a/metron-interface/metron-rest/src/test/resources/README.vm
+++ b/metron-interface/metron-rest/src/test/resources/README.vm
@@ -1,66 +1,83 @@
-#[[#]]# Metron REST and Configuration UI
+#[[#]]# Metron REST
-This UI exposes and aids in sensor configuration.
+This module provides a RESTful API for interacting with Metron.
#[[##]]# Prerequisites
* A running Metron cluster
-* A running instance of MySQL
* Java 8 installed
* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
#[[##]]# Installation
-1. Package the Application with Maven:
- ```
- mvn clean package
- ```
+1. Package the application with Maven:
+```
+mvn clean package
+```
1. Untar the archive in the target directory. The directory structure will look like:
- ```
- bin
- start.sh
- lib
- metron-rest-version.jar
- ```
+```
+bin
+ start_metron_rest.sh
+lib
+ metron-rest-$METRON_VERSION.jar
+```
-1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
- ```
- export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
- ```
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
-1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
- ```
- export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
- ```
+1. Start the application with this command:
+```
+./bin/start_metron_rest.sh /path/to/application.yml
+```
-1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+#[[##]]# Usage
-1. Create a Config UI database in MySQL with this command:
- ```
- CREATE DATABASE IF NOT EXISTS metronrest
- ```
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
-1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+#[[##]]# Security
-1. Start the UI with this command:
- ```
- ./bin/start.sh /path/to/application.yml
- ```
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users:
+```
+./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+```
-#[[##]]# Usage
+For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows:
+
+1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+
+1. Connect to MySQL and create a Metron REST database:
+```
+CREATE DATABASE IF NOT EXISTS metronrest
+```
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement:
+1. Add users:
```
use metronrest;
insert into users (username, password, enabled) values ('your_username','your_password',1);
insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
```
-Users can be added to additional groups with this SQL statement:
+
+1. Replace the H2 connection information in the application.yml file with MySQL connection information:
+```
+spring:
+ datasource:
+ driverClassName: com.mysql.jdbc.Driver
+ url: jdbc:mysql://mysql_host:3306/metronrest
+ username: metron_rest_user
+ password: metron_rest_password
+ platform: mysql
```
-use metronrest;
-insert into authorities (username, authority) values ('your_username', 'your_group');
+
+1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
+```
+<dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.client.version}</version>
+</dependency>
```
+1. Follow the steps in the [Installation](#installation) section
+
#[[##]]# API
Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI.
@@ -86,6 +103,38 @@ Request and Response objects are JSON formatted. The JSON schemas are available
#end
#end
+#[[##]]# Testing
+
+Profiles are includes for both the metron-docker and Quick Dev environments.
+
+#[[###]]# metron-docker
+
+Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=docker,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
+
+#[[###]]# Quick Dev
+
+Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=vagrant,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
+
+To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
+```
+mvn clean package
+scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/
+```
+Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari:
+```
+java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+```
+The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.
#[[##]]# License
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/pom.xml b/metron-interface/pom.xml
index 078da19..805c024 100644
--- a/metron-interface/pom.xml
+++ b/metron-interface/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.3.0</version>
+ <version>0.3.1</version>
</parent>
<description>Interfaces for Metron</description>
<url>https://metron.incubator.apache.org/</url>