You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/31 19:19:58 UTC
[02/50] [abbrv] metron git commit: METRON-1661 Create Pcap Query
Filter endpoint (merrimanr) closes apache/metron#1125
METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes apache/metron#1125
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0e87576
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0e87576
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0e87576
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: d0e87576944b6370eb1d9a0c98d0450f2c51f052
Parents: e78bc53
Author: merrimanr <me...@gmail.com>
Authored: Mon Jul 23 09:09:37 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Mon Jul 23 09:09:37 2018 -0500
----------------------------------------------------------------------
.../rest/model/pcap/FixedPcapRequest.java | 1 +
.../metron/rest/model/pcap/PcapRequest.java | 4 +-
.../rest/model/pcap/QueryPcapOptions.java | 35 +++++++++
.../rest/model/pcap/QueryPcapRequest.java | 59 ++++++++++++++
metron-interface/metron-rest/README.md | 10 ++-
.../metron/rest/controller/PcapController.java | 15 +++-
.../apache/metron/rest/service/PcapService.java | 8 +-
.../rest/service/impl/PcapServiceImpl.java | 13 ++--
.../PcapControllerIntegrationTest.java | 81 +++++++++++++-------
.../apache/metron/rest/mock/MockPcapJob.java | 13 ++--
.../rest/service/impl/PcapServiceImplTest.java | 55 +++++++++++--
11 files changed, 240 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
index a2d345b..d91aac7 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -87,6 +87,7 @@ public class FixedPcapRequest extends PcapRequest {
FixedPcapOptions.INCLUDE_REVERSE.put(this, includeReverse);
}
+ @Override
public void setFields() {
Map<String, String> fields = new HashMap<>();
if (getIpSrcAddr() != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
index 64ed932..44721fa 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
@@ -22,7 +22,7 @@ import org.apache.metron.pcap.config.PcapOptions;
import java.util.HashMap;
-public class PcapRequest extends AbstractMapDecorator<String, Object> {
+public abstract class PcapRequest extends AbstractMapDecorator<String, Object> {
public PcapRequest() {
super(new HashMap<>());
@@ -78,4 +78,6 @@ public class PcapRequest extends AbstractMapDecorator<String, Object> {
public void setNumReducers(Integer numReducers) {
PcapOptions.NUM_REDUCERS.put(this, numReducers);
}
+
+ public abstract void setFields();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
new file mode 100644
index 0000000..1ae3da6
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model.pcap;
+
+import org.apache.metron.common.configuration.ConfigOption;
+
+public enum QueryPcapOptions implements ConfigOption {
+ QUERY("query");
+
+ String key;
+
+ QueryPcapOptions(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
new file mode 100644
index 0000000..0da3e69
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model.pcap;
+
+import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.filter.query.QueryPcapFilter;
+
+public class QueryPcapRequest extends PcapRequest {
+
+ public QueryPcapRequest() {
+ PcapOptions.FILTER_IMPL.put(this, new QueryPcapFilter.Configurator());
+ }
+
+ public String getQuery() {
+ return QueryPcapOptions.QUERY.get(this, String.class);
+ }
+
+ public void setQuery(String query) {
+ QueryPcapOptions.QUERY.put(this, query);
+ }
+
+ @Override
+ public void setFields() {
+ PcapOptions.FIELDS.put(this, getQuery());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ QueryPcapRequest queryPcapRequest = (QueryPcapRequest) o;
+
+ return (super.equals(o)) &&
+ (getQuery() != null ? getQuery().equals(queryPcapRequest.getQuery()) : queryPcapRequest.getQuery() != null);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + (getQuery() != null ? getQuery().hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 4a7102f..53f5978 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -254,6 +254,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
| [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)|
+| [ `GET /api/v1/pcap/query`](#get-apiv1pcapquery)|
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
@@ -502,12 +503,19 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns 'true' if the status changed and 'false' if it did not.
### `POST /api/v1/pcap/fixed`
- * Description: Executes a Fixed Pcap Query.
+ * Description: Executes a Fixed Filter Pcap Query.
* Input:
* fixedPcapRequest - A Fixed Pcap Request which includes fixed filter fields like ip source address and protocol
* Returns:
* 200 - Returns a job status with job ID.
+### `POST /api/v1/pcap/query`
+ * Description: Executes a Query Filter Pcap Query.
+ * Input:
+ * queryPcapRequest - A Query Pcap Request which includes Stellar query field
+ * Returns:
+ * 200 - Returns a job status with job ID.
+
### `POST /api/v1/pcap/{jobId}`
* Description: Gets job status for Pcap query job.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index 23bb0b9..be95718 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -22,11 +22,11 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
+import org.apache.metron.rest.model.pcap.QueryPcapRequest;
import org.apache.metron.rest.security.SecurityUtils;
import org.apache.metron.rest.service.PcapService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -54,12 +54,21 @@ public class PcapController {
@Autowired
private PcapService pcapQueryService;
- @ApiOperation(value = "Executes a Fixed Pcap Query.")
+ @ApiOperation(value = "Executes a Fixed Filter Pcap Query.")
@ApiResponses(value = { @ApiResponse(message = "Returns a job status with job ID.", code = 200)})
@RequestMapping(value = "/fixed", method = RequestMethod.POST)
ResponseEntity<PcapStatus> fixed(@ApiParam(name="fixedPcapRequest", value="A Fixed Pcap Request"
+ " which includes fixed filter fields like ip source address and protocol", required=true)@RequestBody FixedPcapRequest fixedPcapRequest) throws RestException {
- PcapStatus pcapStatus = pcapQueryService.fixed(SecurityUtils.getCurrentUser(), fixedPcapRequest);
+ PcapStatus pcapStatus = pcapQueryService.submit(SecurityUtils.getCurrentUser(), fixedPcapRequest);
+ return new ResponseEntity<>(pcapStatus, HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Executes a Query Filter Pcap Query.")
+ @ApiResponses(value = { @ApiResponse(message = "Returns a job status with job ID.", code = 200)})
+ @RequestMapping(value = "/query", method = RequestMethod.POST)
+ ResponseEntity<PcapStatus> query(@ApiParam(name="queryPcapRequest", value="A Query Pcap Request"
+ + " which includes Stellar query field", required=true)@RequestBody QueryPcapRequest queryPcapRequest) throws RestException {
+ PcapStatus pcapStatus = pcapQueryService.submit(SecurityUtils.getCurrentUser(), queryPcapRequest);
return new ResponseEntity<>(pcapStatus, HttpStatus.OK);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
index f84735d..2ae34a3 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
@@ -19,15 +19,15 @@ package org.apache.metron.rest.service;
import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
-import org.apache.metron.rest.model.pcap.FixedPcapRequest;
-
-import java.io.InputStream;
+import org.apache.metron.rest.model.pcap.PcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
+import java.io.InputStream;
+
public interface PcapService {
- PcapStatus fixed(String username, FixedPcapRequest fixedPcapRequest) throws RestException;
+ PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException;
PcapStatus getJobStatus(String username, String jobId) throws RestException;
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
index e341184..77a8934 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
@@ -18,7 +18,6 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,7 +31,6 @@ import org.apache.metron.pcap.config.PcapOptions;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.config.PcapJobSupplier;
-import org.apache.metron.rest.model.pcap.FixedPcapRequest;
import org.apache.metron.rest.model.pcap.PcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
@@ -43,8 +41,6 @@ import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
@Service
public class PcapServiceImpl implements PcapService {
@@ -65,11 +61,11 @@ public class PcapServiceImpl implements PcapService {
}
@Override
- public PcapStatus fixed(String username, FixedPcapRequest fixedPcapRequest) throws RestException {
+ public PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException {
try {
- setPcapOptions(username, fixedPcapRequest);
- fixedPcapRequest.setFields();
- pcapJobSupplier.setPcapRequest(fixedPcapRequest);
+ setPcapOptions(username, pcapRequest);
+ pcapRequest.setFields();
+ pcapJobSupplier.setPcapRequest(pcapRequest);
JobStatus jobStatus = jobManager.submit(pcapJobSupplier, username);
return jobStatusToPcapStatus(jobStatus);
} catch (IOException | JobException e) {
@@ -149,6 +145,7 @@ public class PcapServiceImpl implements PcapService {
return pdml;
}
+ @Override
public InputStream getRawPcap(String username, String jobId, Integer page) throws RestException {
InputStream inputStream = null;
Path path = getPath(username, jobId, page);
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
index 6189d2c..25956e4 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
@@ -17,37 +17,17 @@
*/
package org.apache.metron.rest.controller;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.hadoop.fs.Path;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.common.Constants;
import org.apache.metron.job.JobStatus;
import org.apache.metron.job.Pageable;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.job.JobStatus;
-import org.apache.metron.job.Pageable;
import org.apache.metron.pcap.PcapHelper;
import org.apache.metron.pcap.PcapPages;
import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
+import org.apache.metron.pcap.filter.query.QueryPcapFilter;
import org.apache.metron.rest.mock.MockPcapJob;
-import org.apache.metron.rest.mock.MockPcapToPdmlScriptWrapper;
-import org.apache.metron.rest.model.PcapResponse;
import org.apache.metron.rest.service.PcapService;
import org.junit.Assert;
import org.junit.Before;
@@ -62,6 +42,22 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Map;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
@@ -101,6 +97,20 @@ public class PcapControllerIntegrationTest {
@Multiline
public static String fixedWithDefaultsJson;
+ /**
+ {
+ "basePath": "/base/path",
+ "baseInterimResultPath": "/base/interim/result/path",
+ "finalOutputPath": "/final/output/path",
+ "startTimeMs": 10,
+ "endTimeMs": 20,
+ "numReducers": 2,
+ "query": "query"
+ }
+ */
+ @Multiline
+ public static String queryJson;
+
@Autowired
private PcapService pcapService;
@@ -123,17 +133,16 @@ public class PcapControllerIntegrationTest {
public void testSecurity() throws Exception {
this.mockMvc.perform(post(pcapUrl + "/fixed").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
.andExpect(status().isUnauthorized());
+
+ this.mockMvc.perform(post(pcapUrl + "/query").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(queryJson))
+ .andExpect(status().isUnauthorized());
}
@Test
public void testFixedRequest() throws Exception {
MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
- List<byte[]> results = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes());
- mockPcapJob.setResults(results);
mockPcapJob.setStatus(new JobStatus().withState(JobStatus.State.RUNNING));
- PcapResponse expectedReponse = new PcapResponse();
- expectedReponse.setPcaps(results);
this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
@@ -187,6 +196,26 @@ public class PcapControllerIntegrationTest {
}
@Test
+ public void testQueryRequest() throws Exception {
+ MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
+ mockPcapJob.setStatus(new JobStatus().withState(JobStatus.State.RUNNING));
+
+ this.mockMvc.perform(post(pcapUrl + "/query").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(queryJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.jobStatus").value("RUNNING"));
+
+ Assert.assertEquals("/base/path", mockPcapJob.getBasePath());
+ Assert.assertEquals("/base/interim/result/path", mockPcapJob.getBaseInterrimResultPath());
+ Assert.assertEquals("/final/output/path", mockPcapJob.getFinalOutputPath());
+ Assert.assertEquals(10000000, mockPcapJob.getStartTimeNs());
+ Assert.assertEquals(20000000, mockPcapJob.getEndTimeNs());
+ Assert.assertEquals(2, mockPcapJob.getNumReducers());
+ Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof QueryPcapFilter.Configurator);
+ Assert.assertEquals("query", mockPcapJob.getQuery());
+ }
+
+ @Test
public void testGetStatus() throws Exception {
MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
index df65635..6a954e8 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
@@ -45,11 +45,10 @@ public class MockPcapJob extends PcapJob<Path> {
private Map<String, String> fixedFields;
private PcapFilterConfigurator filterImpl;
private int recPerFile;
- private SequenceFileIterable sequenceFileIterable;
+ private String query;
private Statusable<Path> statusable;
public MockPcapJob() {
- sequenceFileIterable = mock(SequenceFileIterable.class);
statusable = mock(Statusable.class);
}
@@ -64,6 +63,8 @@ public class MockPcapJob extends PcapJob<Path> {
Object fields = PcapOptions.FIELDS.get(configuration, Object.class);
if (fields instanceof Map) {
this.fixedFields = (Map<String, String>) fields;
+ } else {
+ this.query = (String) fields;
}
this.filterImpl = PcapOptions.FILTER_IMPL.get(configuration, PcapFilterConfigurator.class);
this.recPerFile = PcapOptions.NUM_RECORDS_PER_FILE.get(configuration, Integer.class);
@@ -92,10 +93,6 @@ public class MockPcapJob extends PcapJob<Path> {
when(statusable.isDone()).thenReturn(isDone);
}
- public void setResults(List<byte[]> pcaps) {
- when(sequenceFileIterable.iterator()).thenReturn(pcaps.iterator());
- }
-
public String getBasePath() {
return basePath;
}
@@ -136,6 +133,10 @@ public class MockPcapJob extends PcapJob<Path> {
return fixedFields;
}
+ public String getQuery() {
+ return query;
+ }
+
public PcapFilterConfigurator getFilterImpl() {
return filterImpl;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
index 3c6d506..e9f8f9f 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
@@ -32,12 +32,14 @@ import org.apache.metron.job.manager.InMemoryJobManager;
import org.apache.metron.job.manager.JobManager;
import org.apache.metron.pcap.PcapHelper;
import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
+import org.apache.metron.pcap.filter.query.QueryPcapFilter;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.config.PcapJobSupplier;
import org.apache.metron.rest.mock.MockPcapJob;
import org.apache.metron.rest.mock.MockPcapJobSupplier;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+import org.apache.metron.rest.model.pcap.QueryPcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
import org.hamcrest.CoreMatchers;
@@ -190,7 +192,7 @@ public class PcapServiceImplTest {
}
@Test
- public void fixedShouldProperlyCallPcapJobQuery() throws Exception {
+ public void submitShouldProperlySubmitFixedPcapRequest() throws Exception {
FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
fixedPcapRequest.setBasePath("basePath");
fixedPcapRequest.setBaseInterimResultPath("baseOutputPath");
@@ -232,7 +234,7 @@ public class PcapServiceImplTest {
expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
expectedPcapStatus.setDescription("description");
- Assert.assertEquals(expectedPcapStatus, pcapService.fixed("user", fixedPcapRequest));
+ Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest));
Assert.assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus()));
Assert.assertEquals("basePath", mockPcapJob.getBasePath());
Assert.assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath());
@@ -253,7 +255,7 @@ public class PcapServiceImplTest {
}
@Test
- public void fixedShouldProperlyCallPcapJobQueryWithDefaults() throws Exception {
+ public void submitShouldProperlySubmitWithDefaults() throws Exception {
long beforeJobTime = System.currentTimeMillis();
FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
@@ -275,7 +277,7 @@ public class PcapServiceImplTest {
expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
expectedPcapStatus.setDescription("description");
- Assert.assertEquals(expectedPcapStatus, pcapService.fixed("user", fixedPcapRequest));
+ Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest));
Assert.assertEquals("/base/path", mockPcapJob.getBasePath());
Assert.assertEquals("/base/interim/result/path", mockPcapJob.getBaseInterrimResultPath());
Assert.assertEquals("/final/output/path", mockPcapJob.getFinalOutputPath());
@@ -289,6 +291,49 @@ public class PcapServiceImplTest {
}
@Test
+ public void submitShouldProperlySubmitQueryPcapRequest() throws Exception {
+ QueryPcapRequest queryPcapRequest = new QueryPcapRequest();
+ queryPcapRequest.setBasePath("basePath");
+ queryPcapRequest.setBaseInterimResultPath("baseOutputPath");
+ queryPcapRequest.setFinalOutputPath("finalOutputPath");
+ queryPcapRequest.setStartTimeMs(1L);
+ queryPcapRequest.setEndTimeMs(2L);
+ queryPcapRequest.setNumReducers(2);
+ queryPcapRequest.setQuery("query");
+ MockPcapJob mockPcapJob = new MockPcapJob();
+ mockPcapJobSupplier.setMockPcapJob(mockPcapJob);
+ JobManager jobManager = new InMemoryJobManager<>();
+
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper));
+ FileSystem fileSystem = mock(FileSystem.class);
+ doReturn(fileSystem).when(pcapService).getFileSystem();
+ mockPcapJob.setStatus(new JobStatus()
+ .withJobId("jobId")
+ .withDescription("description")
+ .withPercentComplete(0L)
+ .withState(JobStatus.State.RUNNING));
+
+ String expectedFields = "query";
+ PcapStatus expectedPcapStatus = new PcapStatus();
+ expectedPcapStatus.setJobId("jobId");
+ expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
+ expectedPcapStatus.setDescription("description");
+
+ Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", queryPcapRequest));
+ Assert.assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus()));
+ Assert.assertEquals("basePath", mockPcapJob.getBasePath());
+ Assert.assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath());
+ Assert.assertEquals("finalOutputPath", mockPcapJob.getFinalOutputPath());
+ Assert.assertEquals(1000000, mockPcapJob.getStartTimeNs());
+ Assert.assertEquals(2000000, mockPcapJob.getEndTimeNs());
+ Assert.assertEquals(2, mockPcapJob.getNumReducers());
+ Assert.assertEquals(100, mockPcapJob.getRecPerFile());
+ Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof QueryPcapFilter.Configurator);
+ Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
+ Assert.assertEquals("query", mockPcapJob.getQuery());
+ }
+
+ @Test
public void fixedShouldThrowRestException() throws Exception {
exception.expect(RestException.class);
exception.expectMessage("some job exception");
@@ -301,7 +346,7 @@ public class PcapServiceImplTest {
doReturn(fileSystem).when(pcapService).getFileSystem();
when(jobManager.submit(pcapJobSupplier, "user")).thenThrow(new JobException("some job exception"));
- pcapService.fixed("user", fixedPcapRequest);
+ pcapService.submit("user", fixedPcapRequest);
}
@Test