You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/10/25 18:48:24 UTC
metron git commit: METRON-1243: Add a REST endpoint which allows us
to get a list of all indice closes apache/incubator-metron#797
Repository: metron
Updated Branches:
refs/heads/master a25de0fc1 -> 1c12e22df
METRON-1243: Add a REST endpoint which allows us to get a list of all indice closes apache/incubator-metron#797
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c12e22d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c12e22d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c12e22d
Branch: refs/heads/master
Commit: 1c12e22df03b53981ca8cef426ab26f263f289c0
Parents: a25de0f
Author: cstella <ce...@gmail.com>
Authored: Wed Oct 25 14:48:17 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Wed Oct 25 14:48:17 2017 -0400
----------------------------------------------------------------------
.../SensorIndexingConfigController.java | 8 ++
.../service/SensorIndexingConfigService.java | 2 +
.../impl/SensorIndexingConfigServiceImpl.java | 34 +++++++-
...IndexingConfigControllerIntegrationTest.java | 15 ++++
.../SensorIndexingConfigServiceImplTest.java | 83 +++++++++++++++++++-
5 files changed, 140 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
index ffd80fb..ff1d6bd 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java
@@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
+import java.util.List;
import java.util.Map;
@RestController
@@ -74,6 +75,13 @@ public class SensorIndexingConfigController {
return new ResponseEntity<>(sensorIndexingConfigService.getAll(), HttpStatus.OK);
}
+ @ApiOperation(value = "Retrieves all indices")
+ @ApiResponse(message = "Returns all the indices in use", code = 200)
+ @RequestMapping(value = "/list/indices/{writerName}", method = RequestMethod.GET)
+ ResponseEntity<Iterable<String>> getAllIndices(@ApiParam(name="writerName", value="Writer name. One of solr, elasticsearch or hdfs", required=true)@PathVariable String writerName) throws Exception {
+ return new ResponseEntity<>(sensorIndexingConfigService.getAllIndices(writerName), HttpStatus.OK);
+ }
+
@ApiOperation(value = "Deletes a SensorIndexingConfig from Zookeeper")
@ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig was deleted", code = 200),
@ApiResponse(message = "SensorIndexingConfig is missing", code = 404) })
http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
index 0699431..0f32e53 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java
@@ -32,6 +32,8 @@ public interface SensorIndexingConfigService {
List<String> getAllTypes() throws RestException;
+ Iterable<String> getAllIndices(String writerName) throws RestException;
+
boolean delete(String name) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index 5c73b26..a90d123 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -18,14 +18,16 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.SensorIndexingConfigService;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -33,6 +35,9 @@ import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Collections;
+import java.util.Set;
+import java.util.HashSet;
@Service
public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
@@ -85,6 +90,33 @@ public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigServ
return configs.getTypes();
}
+ /**
+ * Get a list of index names for a given writer (e.g. elasticsearch, solr, hdfs).
+ * This functions in the following way:
+ * * If an index config exists, then the index name will be returned. If unspecified, then the sensor name is used
+ * * If a parser exists and an index does NOT exist, then it will be included.
+ * * If the writer is disabled in the index config, then it will NOT be included.
+ * @param writerName The writer name to use
+ * @return An iterable of index names
+ * @throws RestException
+ */
+ @Override
+ public Iterable<String> getAllIndices(String writerName) throws RestException {
+ if(StringUtils.isEmpty(writerName)) {
+ return Collections.emptyList();
+ }
+ IndexingConfigurations indexingConfigs = cache.get( IndexingConfigurations.class);
+ ParserConfigurations parserConfigs = cache.get( ParserConfigurations.class);
+ Set<String> ret = new HashSet<>();
+ for(String sensorName : Iterables.concat(parserConfigs.getTypes(), indexingConfigs.getTypes())) {
+ if(indexingConfigs.isEnabled(sensorName, writerName)) {
+ String indexName = indexingConfigs.getIndex(sensorName, writerName);
+ ret.add(indexName == null ? sensorName : indexName);
+ }
+ }
+ return ret;
+ }
+
@Override
public boolean delete(String name) throws RestException {
try {
http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index 674c55a..28977fd 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -98,12 +98,27 @@ public class SensorIndexingConfigControllerIntegrationTest {
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(content().bytes("{}".getBytes()));
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().bytes("[]".getBytes()));
+
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/blah").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().bytes("[]".getBytes()));
+
this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
.andExpect(jsonPath("$.index").value("broTest"))
.andExpect(jsonPath("$.batchSize").value(1));
+ this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().bytes("[\"broTest\"]".getBytes()));
+
assertEventually(() -> this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
http://git-wip-us.apache.org/repos/asf/metron/blob/1c12e22d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
index 9641a52..1d8e3f5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java
@@ -17,10 +17,10 @@
*/
package org.apache.metron.rest.service.impl;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
@@ -29,12 +29,14 @@ import org.apache.curator.framework.api.GetDataBuilder;
import org.apache.curator.framework.api.SetDataBuilder;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.SensorIndexingConfigService;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -44,6 +46,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Collections;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -139,7 +142,85 @@ public class SensorIndexingConfigServiceImplTest {
}
+ @Test
+ public void getAllIndicesWithOnlyParsers() throws RestException {
+ ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+ when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort"));
+ IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+ when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
+ when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null);
+ when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
+ when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+
+ when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+ when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+ List<String> indices = new ArrayList<String>();
+ Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+ Assert.assertEquals(2, indices.size());
+ Assert.assertTrue(indices.contains("bro"));
+ Assert.assertTrue(indices.contains("snort"));
+ }
+
+ @Test
+ public void getAllIndicesWithOnlyIndexing() throws RestException {
+ ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+ when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList());
+ IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+ // rename bro, include snort by default configs, and disable yaf
+ when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "yaf"));
+ when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
+ when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
+ when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(false);
+
+ when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+ when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+ List<String> indices = new ArrayList<String>();
+ Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+ Assert.assertEquals(2, indices.size());
+ Assert.assertTrue(indices.contains("renamed_bro"));
+ Assert.assertTrue(indices.contains("snort"));
+ }
+ @Test
+ public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
+
+ ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+ when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
+ IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+ when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
+ when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
+ when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
+ when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
+ when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
+ when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
+ when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+ when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+ List<String> indices = new ArrayList<String>();
+ Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+ Assert.assertEquals(3, indices.size());
+ Assert.assertTrue(indices.contains("renamed_bro"));
+ Assert.assertTrue(indices.contains("snort"));
+ Assert.assertTrue(indices.contains("yaf"));
+ }
+
+ @Test
+ public void getAllIndicesWithNoConfigs() throws RestException {
+ ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
+ when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList());
+ IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
+ when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
+
+ when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
+ when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
+ List<String> indices = new ArrayList<String>();
+ Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
+ Assert.assertEquals(0, indices.size());
+ }
@Test
public void getAllTypesShouldProperlyReturnTypes() throws Exception {