You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/08/17 15:34:25 UTC
[10/51] [abbrv] metron git commit: METRON-1685 Retrieve Pcap results
in raw binary format (merrimanr) closes apache/metron#1123
METRON-1685 Retrieve Pcap results in raw binary format (merrimanr) closes apache/metron#1123
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a5a51399
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a5a51399
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a5a51399
Branch: refs/heads/master
Commit: a5a51399d2eafd2535d79bb13ee0d4d8eb2e2d23
Parents: 3e5ef41
Author: merrimanr <me...@gmail.com>
Authored: Fri Jul 20 09:37:34 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Fri Jul 20 09:37:34 2018 -0500
----------------------------------------------------------------------
metron-interface/metron-rest/README.md | 10 ++++
.../metron/rest/controller/PcapController.java | 40 ++++++++++++++
.../apache/metron/rest/service/PcapService.java | 5 +-
.../rest/service/impl/PcapServiceImpl.java | 17 ++++++
.../PcapControllerIntegrationTest.java | 44 ++++++++++++++++
.../rest/service/impl/PcapServiceImplTest.java | 55 ++++++++++++++++++++
6 files changed, 170 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 7b3a263..4a7102f 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -256,6 +256,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)|
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
+| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
@@ -523,6 +524,15 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Returns PDML in json format.
* 404 - Job or page is missing.
+
+### `POST /api/v1/pcap/{jobId}/raw`
+ * Description: Download Pcap Results for a page.
+ * Input:
+ * jobId - Job ID of submitted job
+ * page - Page number
+ * Returns:
+ * 200 - Returns Pcap as a file download.
+ * 404 - Job or page is missing.
### `POST /api/v1/search/search`
* Description: Searches the indexing store. GUIDs must be quoted to ensure correct results.
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index 47bc6a0..23bb0b9 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -21,6 +21,8 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
@@ -37,10 +39,18 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
@RestController
@RequestMapping("/api/v1/pcap")
public class PcapController {
+ private static final String PCAP_FILENAME_FORMAT = "pcap_%s_%s.pcap";
+
@Autowired
private PcapService pcapQueryService;
@@ -99,4 +109,34 @@ public class PcapController {
}
}
+ @ApiOperation(value = "Download Pcap Results for a page.")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Returns Pcap as a file download.", code = 200),
+ @ApiResponse(message = "Job or page is missing.", code = 404)
+ })
+ @RequestMapping(value = "/{jobId}/raw", method = RequestMethod.GET)
+ void raw(@ApiParam(name="jobId", value="Job ID of submitted job", required=true)@PathVariable String jobId,
+ @ApiParam(name="page", value="Page number", required=true)@RequestParam Integer page,
+ @RequestParam(defaultValue = "", required = false) String fileName,
+ final HttpServletRequest request, final HttpServletResponse response) throws RestException {
+ try (InputStream inputStream = pcapQueryService.getRawPcap(SecurityUtils.getCurrentUser(), jobId, page);
+ OutputStream output = response.getOutputStream()) {
+ response.reset();
+ if (inputStream == null) {
+ response.setStatus(HttpStatus.NOT_FOUND.value());
+ } else {
+ response.setContentType("application/octet-stream");
+ if (fileName.isEmpty()) {
+ fileName = String.format(PCAP_FILENAME_FORMAT, jobId, page);
+ }
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + fileName + "\"");
+ int size = IOUtils.copy(inputStream, output);
+ response.setContentLength(size);
+ output.flush();
+ }
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
index 9421ce3..f84735d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
@@ -20,9 +20,10 @@ package org.apache.metron.rest.service;
import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+
+import java.io.InputStream;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
-import org.apache.metron.rest.model.pcap.Pdml;
public interface PcapService {
@@ -35,4 +36,6 @@ public interface PcapService {
Path getPath(String username, String jobId, Integer page) throws RestException;
Pdml getPdml(String username, String jobId, Integer page) throws RestException;
+
+ InputStream getRawPcap(String username, String jobId, Integer page) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
index 7894b1a..e341184 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +43,8 @@ import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
@Service
public class PcapServiceImpl implements PcapService {
@@ -146,6 +149,20 @@ public class PcapServiceImpl implements PcapService {
return pdml;
}
+ public InputStream getRawPcap(String username, String jobId, Integer page) throws RestException {
+ InputStream inputStream = null;
+ Path path = getPath(username, jobId, page);
+ try {
+ FileSystem fileSystem = getFileSystem();
+ if (path!= null && fileSystem.exists(path)) {
+ inputStream = fileSystem.open(path);
+ }
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ return inputStream;
+ }
+
protected void setPcapOptions(String username, PcapRequest pcapRequest) throws IOException {
PcapOptions.JOB_NAME.put(pcapRequest, "jobName");
PcapOptions.USERNAME.put(pcapRequest, username);
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
index 2fa64cd..6189d2c 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
@@ -25,21 +25,28 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.fs.Path;
+import org.apache.commons.io.FileUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.job.JobStatus;
import org.apache.metron.job.Pageable;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.job.JobStatus;
+import org.apache.metron.job.Pageable;
import org.apache.metron.pcap.PcapHelper;
import org.apache.metron.pcap.PcapPages;
import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
import org.apache.metron.rest.mock.MockPcapJob;
+import org.apache.metron.rest.mock.MockPcapToPdmlScriptWrapper;
import org.apache.metron.rest.model.PcapResponse;
import org.apache.metron.rest.service.PcapService;
import org.junit.Assert;
@@ -301,5 +308,42 @@ public class PcapControllerIntegrationTest {
.andExpect(status().isNotFound());
}
+ @Test
+ public void testRawDownload() throws Exception {
+ String pcapFileContents = "pcap file contents";
+ FileUtils.write(new File("./target/pcapFile"), pcapFileContents, "UTF8");
+
+ MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
+
+ mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING));
+
+ this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.jobId").value("jobId"))
+ .andExpect(jsonPath("$.jobStatus").value("RUNNING"));
+
+ Pageable<Path> pageable = new PcapPages(Arrays.asList(new Path("./target/pcapFile")));
+ mockPcapJob.setIsDone(true);
+ mockPcapJob.setPageable(pageable);
+
+ this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=1").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(header().string("Content-Disposition", "attachment; filename=\"pcap_jobId_1.pcap\""))
+ .andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length())))
+ .andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream")))
+ .andExpect(content().bytes(pcapFileContents.getBytes()));
+
+ this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=1&fileName=pcapFile.pcap").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(header().string("Content-Disposition", "attachment; filename=\"pcapFile.pcap\""))
+ .andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length())))
+ .andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream")))
+ .andExpect(content().bytes(pcapFileContents.getBytes()));
+
+ this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=2").with(httpBasic(user, password)))
+ .andExpect(status().isNotFound());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
index d818c77..3c6d506 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.service.impl;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.metron.common.Constants;
@@ -471,4 +472,58 @@ public class PcapServiceImplTest {
pcapService.getPdml("user", "jobId", 1);
}
+ @Test
+ public void getRawShouldProperlyReturnInputStream() throws Exception {
+ FSDataInputStream inputStream = mock(FSDataInputStream.class);
+ Path path = new Path("./target");
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), new PcapToPdmlScriptWrapper()));
+ FileSystem fileSystem = mock(FileSystem.class);
+ doReturn(fileSystem).when(pcapService).getFileSystem();
+ when(fileSystem.exists(path)).thenReturn(true);
+ doReturn(path).when(pcapService).getPath("user", "jobId", 1);
+ when(fileSystem.open(path)).thenReturn(inputStream);
+
+ Assert.assertEquals(inputStream, pcapService.getRawPcap("user", "jobId", 1));
+ }
+
+ @Test
+ public void getRawShouldReturnNullOnInvalidPage() throws Exception {
+ Path path = new Path("/some/path");
+
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
+ FileSystem fileSystem = mock(FileSystem.class);
+ doReturn(fileSystem).when(pcapService).getFileSystem();
+
+ assertNull(pcapService.getRawPcap("user", "jobId", 1));
+ }
+
+ @Test
+ public void getRawShouldReturnNullOnNonexistentPath() throws Exception {
+ Path path = new Path("/some/path");
+
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
+ FileSystem fileSystem = mock(FileSystem.class);
+ doReturn(fileSystem).when(pcapService).getFileSystem();
+ when(fileSystem.exists(path)).thenReturn(false);
+ doReturn(path).when(pcapService).getPath("user", "jobId", 1);
+
+ assertNull(pcapService.getRawPcap("user", "jobId", 1));
+ }
+
+ @Test
+ public void getRawShouldThrowException() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("some exception");
+
+ Path path = new Path("./target");
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
+ FileSystem fileSystem = mock(FileSystem.class);
+ doReturn(fileSystem).when(pcapService).getFileSystem();
+ when(fileSystem.exists(path)).thenReturn(true);
+ doReturn(path).when(pcapService).getPath("user", "jobId", 1);
+ when(fileSystem.open(path)).thenThrow(new IOException("some exception"));
+
+ pcapService.getRawPcap("user", "jobId", 1);
+ }
+
}