You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/31 19:20:01 UTC
[05/50] [abbrv] metron git commit: METRON-1691: REST should limit the
number of Pcap jobs a user can submit (merrimanr via mmiklavc) closes
apache/metron#1129
METRON-1691: REST should limit the number of Pcap jobs a user can submit (merrimanr via mmiklavc) closes apache/metron#1129
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6c90724d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6c90724d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6c90724d
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 6c90724d8d4f06c453128f860eff51037f1870d8
Parents: f1f5dda
Author: merrimanr <me...@gmail.com>
Authored: Wed Jul 25 10:58:05 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Jul 25 10:58:05 2018 -0600
----------------------------------------------------------------------
.../apache/metron/rest/model/pcap/Field.java | 36 ++++-------
.../rest/model/pcap/FixedPcapRequest.java | 30 ---------
.../apache/metron/rest/model/pcap/Packet.java | 8 +--
.../metron/rest/model/pcap/PcapStatus.java | 24 +++----
.../org/apache/metron/rest/model/pcap/Pdml.java | 21 +++---
.../apache/metron/rest/model/pcap/Proto.java | 24 +++----
.../rest/model/pcap/QueryPcapRequest.java | 18 ------
metron-interface/metron-rest/README.md | 27 ++++++--
.../apache/metron/rest/MetronRestConstants.java | 1 +
.../metron/rest/controller/PcapController.java | 13 ++++
.../apache/metron/rest/service/PcapService.java | 6 ++
.../rest/service/impl/PcapServiceImpl.java | 64 ++++++++++++++++--
.../src/main/resources/application.yml | 1 +
.../PcapControllerIntegrationTest.java | 46 +++++++++++++
.../rest/service/impl/PcapServiceImplTest.java | 68 ++++++++++++++++++++
.../metron/job/manager/InMemoryJobManager.java | 4 ++
16 files changed, 261 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
index 9c2878b..4ed71c3 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
public class Field {
@@ -132,33 +133,22 @@ public class Field {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Field field = (Field) o;
-
- return (getName() != null ? getName().equals(field.getName()) : field.getName() != null) &&
- (getPos() != null ? getPos().equals(field.getPos()) : field.getPos() == null) &&
- (getShowname() != null ? getShowname().equals(field.getShowname()) : field.getShowname() == null) &&
- (getSize() != null ? getSize().equals(field.getSize()) : field.getSize() == null) &&
- (getValue() != null ? getValue().equals(field.getValue()) : field.getValue() == null) &&
- (getShow() != null ? getShow().equals(field.getShow()) : field.getShow() == null) &&
- (getUnmaskedvalue() != null ? getUnmaskedvalue().equals(field.getUnmaskedvalue()) : field.getUnmaskedvalue() == null) &&
- (getHide() != null ? getHide().equals(field.getHide()) : field.getHide() == null) &&
- (getFields() != null ? getFields().equals(field.getFields()) : field.getFields() == null) &&
- (getProtos() != null ? getProtos().equals(field.getProtos()) : field.getProtos() == null);
+ return Objects.equals(name, field.name) &&
+ Objects.equals(pos, field.pos) &&
+ Objects.equals(showname, field.showname) &&
+ Objects.equals(size, field.size) &&
+ Objects.equals(value, field.value) &&
+ Objects.equals(show, field.show) &&
+ Objects.equals(unmaskedvalue, field.unmaskedvalue) &&
+ Objects.equals(hide, field.hide) &&
+ Objects.equals(fields, field.fields) &&
+ Objects.equals(protos, field.protos);
}
@Override
public int hashCode() {
- int result = getName() != null ? getName().hashCode() : 0;
- result = 31 * result + (getPos() != null ? getPos().hashCode() : 0);
- result = 31 * result + (getShowname() != null ? getShowname().hashCode() : 0);
- result = 31 * result + (getSize() != null ? getSize().hashCode() : 0);
- result = 31 * result + (getValue() != null ? getValue().hashCode() : 0);
- result = 31 * result + (getShow() != null ? getShow().hashCode() : 0);
- result = 31 * result + (getUnmaskedvalue() != null ? getUnmaskedvalue().hashCode() : 0);
- result = 31 * result + (getHide() != null ? getHide().hashCode() : 0);
- result = 31 * result + (getFields() != null ? getFields().hashCode() : 0);
- result = 31 * result + (getProtos() != null ? getProtos().hashCode() : 0);
- return result;
+
+ return Objects.hash(name, pos, showname, size, value, show, unmaskedvalue, hide, fields, protos);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
index d91aac7..38d05b7 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -113,34 +113,4 @@ public class FixedPcapRequest extends PcapRequest {
}
PcapOptions.FIELDS.put(this, fields);
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- FixedPcapRequest fixedPcapRequest = (FixedPcapRequest) o;
-
- return (super.equals(o)) &&
- (getIpSrcAddr() != null ? getIpSrcAddr().equals(fixedPcapRequest.getIpSrcAddr()) : fixedPcapRequest.getIpSrcAddr() != null) &&
- (getIpDstAddr() != null ? getIpDstAddr().equals(fixedPcapRequest.getIpDstAddr()) : fixedPcapRequest.getIpDstAddr() != null) &&
- (getIpSrcPort() != null ? getIpSrcPort().equals(fixedPcapRequest.getIpSrcPort()) : fixedPcapRequest.getIpSrcPort() != null) &&
- (getIpDstPort() != null ? getIpDstPort().equals(fixedPcapRequest.getIpDstPort()) : fixedPcapRequest.getIpDstPort() != null) &&
- (getProtocol() != null ? getProtocol().equals(fixedPcapRequest.getProtocol()) : fixedPcapRequest.getProtocol() != null) &&
- (getPacketFilter() != null ? getPacketFilter().equals(fixedPcapRequest.getPacketFilter()) : fixedPcapRequest.getPacketFilter() != null) &&
- (getIncludeReverse() != null ? getIncludeReverse().equals(fixedPcapRequest.getIncludeReverse()) : fixedPcapRequest.getIncludeReverse() != null);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + (getIpSrcAddr() != null ? getIpSrcAddr().hashCode() : 0);
- result = 31 * result + (getIpDstAddr() != null ? getIpDstAddr().hashCode() : 0);
- result = 31 * result + (getIpSrcPort() != null ? getIpSrcPort().hashCode() : 0);
- result = 31 * result + (getIpDstPort() != null ? getIpDstPort().hashCode() : 0);
- result = 31 * result + (getProtocol() != null ? getProtocol().hashCode() : 0);
- result = 31 * result + (getPacketFilter() != null ? getPacketFilter().hashCode() : 0);
- result = 31 * result + (getIncludeReverse() != null ? getIncludeReverse().hashCode() : 0);
- return result;
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java
index de21e6b..1773272 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Packet.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.List;
+import java.util.Objects;
public class Packet {
@@ -40,14 +41,13 @@ public class Packet {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Packet packet = (Packet) o;
-
- return (getProtos() != null ? getProtos().equals(packet.getProtos()) : packet.getProtos() == null);
+ return Objects.equals(protos, packet.protos);
}
@Override
public int hashCode() {
- return getProtos() != null ? getProtos().hashCode() : 0;
+
+ return Objects.hash(protos);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java
index f004eb5..43c77fd 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapStatus.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.rest.model.pcap;
+import java.util.Objects;
+
public class PcapStatus {
private String jobId;
@@ -69,23 +71,17 @@ public class PcapStatus {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
- PcapStatus pcapStatus = (PcapStatus) o;
-
- return (getJobId() != null ? getJobId().equals(pcapStatus.getJobId()) : pcapStatus.getJobId() != null) &&
- (getJobStatus() != null ? getJobStatus().equals(pcapStatus.getJobStatus()) : pcapStatus.getJobStatus() != null) &&
- (getDescription() != null ? getDescription().equals(pcapStatus.getDescription()) : pcapStatus.getDescription() != null) &&
- (getPercentComplete() != null ? getPercentComplete().equals(pcapStatus.getPercentComplete()) : pcapStatus.getPercentComplete() != null) &&
- (getPageTotal() != null ? getPageTotal().equals(pcapStatus.getPageTotal()) : pcapStatus.getPageTotal() != null);
+ PcapStatus that = (PcapStatus) o;
+ return Objects.equals(jobId, that.jobId) &&
+ Objects.equals(jobStatus, that.jobStatus) &&
+ Objects.equals(description, that.description) &&
+ Objects.equals(percentComplete, that.percentComplete) &&
+ Objects.equals(pageTotal, that.pageTotal);
}
@Override
public int hashCode() {
- int result = (getJobId() != null ? getJobId().hashCode() : 0);
- result = 31 * result + (getJobStatus() != null ? getJobStatus().hashCode() : 0);
- result = 31 * result + (getDescription() != null ? getDescription().hashCode() : 0);
- result = 31 * result + (getPercentComplete() != null ? getPercentComplete().hashCode() : 0);
- result = 31 * result + (getPageTotal() != null ? getPageTotal().hashCode() : 0);
- return result;
+
+ return Objects.hash(jobId, jobStatus, description, percentComplete, pageTotal);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java
index f44f96b..f59586a 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Pdml.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.List;
+import java.util.Objects;
public class Pdml {
@@ -81,23 +82,17 @@ public class Pdml {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Pdml pdml = (Pdml) o;
-
- return (getVersion() != null ? getVersion().equals(pdml.getVersion()) : pdml.getVersion() != null) &&
- (getCreator() != null ? getCreator().equals(pdml.getCreator()) : pdml.getCreator() == null) &&
- (getTime() != null ? getTime().equals(pdml.getTime()) : pdml.getTime() == null) &&
- (getCaptureFile() != null ? getCaptureFile().equals(pdml.getCaptureFile()) : pdml.getCaptureFile() == null) &&
- (getPackets() != null ? getPackets().equals(pdml.getPackets()) : pdml.getPackets() == null);
+ return Objects.equals(version, pdml.version) &&
+ Objects.equals(creator, pdml.creator) &&
+ Objects.equals(time, pdml.time) &&
+ Objects.equals(captureFile, pdml.captureFile) &&
+ Objects.equals(packets, pdml.packets);
}
@Override
public int hashCode() {
- int result = getVersion() != null ? getVersion().hashCode() : 0;
- result = 31 * result + (getCreator() != null ? getCreator().hashCode() : 0);
- result = 31 * result + (getTime() != null ? getTime().hashCode() : 0);
- result = 31 * result + (getCaptureFile() != null ? getCaptureFile().hashCode() : 0);
- result = 31 * result + (getPackets() != null ? getPackets().hashCode() : 0);
- return result;
+
+ return Objects.hash(version, creator, time, captureFile, packets);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java
index bdd5c1f..2c145a3 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Proto.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.List;
+import java.util.Objects;
public class Proto {
@@ -90,25 +91,18 @@ public class Proto {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Proto proto = (Proto) o;
-
- return (getName() != null ? getName().equals(proto.getName()) : proto.getName() != null) &&
- (getPos() != null ? getPos().equals(proto.getPos()) : proto.getPos() == null) &&
- (getShowname() != null ? getShowname().equals(proto.getShowname()) : proto.getShowname() == null) &&
- (getSize() != null ? getSize().equals(proto.getSize()) : proto.getSize() == null) &&
- (getHide() != null ? getHide().equals(proto.getHide()) : proto.getHide() == null) &&
- (getFields() != null ? getFields().equals(proto.getFields()) : proto.getFields() == null);
+ return Objects.equals(name, proto.name) &&
+ Objects.equals(pos, proto.pos) &&
+ Objects.equals(showname, proto.showname) &&
+ Objects.equals(size, proto.size) &&
+ Objects.equals(hide, proto.hide) &&
+ Objects.equals(fields, proto.fields);
}
@Override
public int hashCode() {
- int result = getName() != null ? getName().hashCode() : 0;
- result = 31 * result + (getPos() != null ? getPos().hashCode() : 0);
- result = 31 * result + (getShowname() != null ? getShowname().hashCode() : 0);
- result = 31 * result + (getSize() != null ? getSize().hashCode() : 0);
- result = 31 * result + (getHide() != null ? getHide().hashCode() : 0);
- result = 31 * result + (getFields() != null ? getFields().hashCode() : 0);
- return result;
+
+ return Objects.hash(name, pos, showname, size, hide, fields);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
index 0da3e69..1a03fda 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java
@@ -38,22 +38,4 @@ public class QueryPcapRequest extends PcapRequest {
public void setFields() {
PcapOptions.FIELDS.put(this, getQuery());
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- QueryPcapRequest queryPcapRequest = (QueryPcapRequest) o;
-
- return (super.equals(o)) &&
- (getQuery() != null ? getQuery().equals(queryPcapRequest.getQuery()) : queryPcapRequest.getQuery() != null);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + (getQuery() != null ? getQuery().hashCode() : 0);
- return result;
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 53f5978..2d9a535 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -212,7 +212,7 @@ METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
## Pcap Query
-The REST application exposes endpoints for querying Pcap data. For more information about filtering options see [Query Filter Utility](/metron-platform/metron-pcap-backend#query-filter-utility).
+The REST application exposes endpoints for querying Pcap data. For more information about filtering options see [Query Filter Utility](../../metron-platform/metron-pcap-backend#query-filter-utility).
There is an endpoint available that will return Pcap data in [PDML](https://wiki.wireshark.org/PDML) format. [Wireshark](https://www.wireshark.org/) must be installed for this feature to work.
Installing wireshark in CentOS can be done with `yum -y install wireshark`.
@@ -253,8 +253,9 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
| [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
| [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
-| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)|
-| [ `GET /api/v1/pcap/query`](#get-apiv1pcapquery)|
+| [ `POST /api/v1/pcap/fixed`](#post-apiv1pcapfixed)|
+| [ `POST /api/v1/pcap/query`](#post-apiv1pcapquery)|
+| [ `GET /api/v1/pcap`](#get-apiv1pcap)|
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
@@ -516,7 +517,14 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Returns a job status with job ID.
-### `POST /api/v1/pcap/{jobId}`
+### `GET /api/v1/pcap`
+ * Description: Gets a list of job statuses for Pcap query jobs that match the requested state.
+ * Input:
+ * state - Job state
+ * Returns:
+ * 200 - Returns a list of job statuses for jobs that match the requested state.
+
+### `GET /api/v1/pcap/{jobId}`
* Description: Gets job status for Pcap query job.
* Input:
* jobId - Job ID of submitted job
@@ -524,7 +532,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns a job status for the Job ID.
* 404 - Job is missing.
-### `POST /api/v1/pcap/{jobId}/pdml`
+### `GET /api/v1/pcap/{jobId}/pdml`
* Description: Gets Pcap Results for a page in PDML format.
* Input:
* jobId - Job ID of submitted job
@@ -533,7 +541,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns PDML in json format.
* 404 - Job or page is missing.
-### `POST /api/v1/pcap/{jobId}/raw`
+### `GET /api/v1/pcap/{jobId}/raw`
* Description: Download Pcap Results for a page.
* Input:
* jobId - Job ID of submitted job
@@ -541,6 +549,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Returns Pcap as a file download.
* 404 - Job or page is missing.
+
+### `DELETE /api/v1/pcap/kill/{jobId}`
+ * Description: Kills running job.
+ * Input:
+ * jobId - Job ID of submitted job
+ * Returns:
+ * 200 - Kills passed job.
### `POST /api/v1/search/search`
* Description: Searches the indexing store. GUIDs must be quoted to ensure correct results.
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index b65d037..d38aedb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -72,6 +72,7 @@ public class MetronRestConstants {
public static final String USER_SETTINGS_HBASE_TABLE_SPRING_PROPERTY = "user.settings.table";
public static final String USER_SETTINGS_HBASE_CF_SPRING_PROPERTY = "user.settings.cf";
+ public static final String USER_JOB_LIMIT_SPRING_PROPERTY = "user.job.limit";
public static final String LOGGING_SYSTEM_PROPERTY = "org.springframework.boot.logging.LoggingSystem";
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index be95718..13a623a 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import org.apache.commons.io.IOUtils;
+import org.apache.metron.job.JobStatus;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
@@ -44,6 +45,7 @@ import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.List;
@RestController
@RequestMapping("/api/v1/pcap")
@@ -87,6 +89,17 @@ public class PcapController {
}
}
+ @ApiOperation(value = "Gets a list of job statuses for Pcap query jobs that match the requested state.")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Returns a list of job statuses for jobs that match the requested state.", code = 200)
+ })
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<List<PcapStatus>> getStatuses(@ApiParam(name="state", value="Job state", required=true)@RequestParam String state) throws RestException {
+
+ List<PcapStatus> jobs = pcapQueryService.getJobStatus(SecurityUtils.getCurrentUser(), JobStatus.State.valueOf(state));
+ return new ResponseEntity<>(jobs, HttpStatus.OK);
+ }
+
@ApiOperation(value = "Gets Pcap Results for a page in PDML format.")
@ApiResponses(value = {
@ApiResponse(message = "Returns PDML in json format.", code = 200),
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
index 2ae34a3..00efab9 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
@@ -18,12 +18,14 @@
package org.apache.metron.rest.service;
import org.apache.hadoop.fs.Path;
+import org.apache.metron.job.JobStatus;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.PcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
import org.apache.metron.rest.model.pcap.Pdml;
import java.io.InputStream;
+import java.util.List;
public interface PcapService {
@@ -31,6 +33,8 @@ public interface PcapService {
PcapStatus getJobStatus(String username, String jobId) throws RestException;
+ List<PcapStatus> getJobStatus(String username, JobStatus.State state) throws RestException;
+
PcapStatus killJob(String username, String jobId) throws RestException;
Path getPath(String username, String jobId, Integer page) throws RestException;
@@ -38,4 +42,6 @@ public interface PcapService {
Pdml getPdml(String username, String jobId, Integer page) throws RestException;
InputStream getRawPcap(String username, String jobId, Integer page) throws RestException;
+
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
index 77a8934..ff80c8f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
@@ -41,6 +41,9 @@ import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
@Service
public class PcapServiceImpl implements PcapService {
@@ -62,6 +65,14 @@ public class PcapServiceImpl implements PcapService {
@Override
public PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException {
+ List<PcapStatus> runningJobs = getJobStatus(username, JobStatus.State.RUNNING);
+ Integer userJobLimit = environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1);
+ if (runningJobs != null && runningJobs.size() >= userJobLimit) {
+ String jobIds = runningJobs.stream().map(PcapStatus::getJobId).collect(Collectors.joining(", "));
+ String message = String.format("Cannot submit job because a job is already running. " +
+ "Please contact the administrator to cancel job(s) with id(s) %s", jobIds);
+ throw new RestException(message);
+ }
try {
setPcapOptions(username, pcapRequest);
pcapRequest.setFields();
@@ -79,13 +90,7 @@ public class PcapServiceImpl implements PcapService {
try {
Statusable<Path> statusable = jobManager.getJob(username, jobId);
if (statusable != null) {
- pcapStatus = jobStatusToPcapStatus(statusable.getStatus());
- if (statusable.isDone()) {
- Pageable<Path> pageable = statusable.get();
- if (pageable != null) {
- pcapStatus.setPageTotal(pageable.getSize());
- }
- }
+ pcapStatus = statusableToPcapStatus(statusable);
}
} catch (JobNotFoundException | InterruptedException e) {
// do nothing and return null pcapStatus
@@ -96,6 +101,40 @@ public class PcapServiceImpl implements PcapService {
}
@Override
+ public List<PcapStatus> getJobStatus(String username, JobStatus.State state) throws RestException {
+ List<PcapStatus> pcapStatuses = new ArrayList<>();
+ try {
+ List<Statusable<Path>> statusables = jobManager.getJobs(username);
+ if (statusables != null) {
+ pcapStatuses = statusables.stream()
+ .filter(statusable -> {
+ try {
+ return statusable.getStatus().getState() == state;
+ } catch (JobException e) {
+ return JobStatus.State.FAILED == state;
+ }
+ })
+ .map(statusable -> {
+ try {
+ return statusableToPcapStatus(statusable);
+ } catch (JobException | InterruptedException e) {
+ PcapStatus pcapStatus = new PcapStatus();
+ pcapStatus.setJobStatus(JobStatus.State.FAILED.toString());
+ pcapStatus.setDescription(e.getMessage());
+ return pcapStatus;
+ }
+ })
+ .collect(Collectors.toList());
+ }
+ } catch (JobNotFoundException e) {
+ // do nothing and return null pcapStatus
+ } catch (JobException e) {
+ throw new RestException(e);
+ }
+ return pcapStatuses;
+ }
+
+ @Override
public PcapStatus killJob(String username, String jobId) throws RestException {
try {
jobManager.killJob(username, jobId);
@@ -183,6 +222,17 @@ public class PcapServiceImpl implements PcapService {
return FileSystem.get(configuration);
}
+ protected PcapStatus statusableToPcapStatus(Statusable<Path> statusable) throws JobException, InterruptedException {
+ PcapStatus pcapStatus = jobStatusToPcapStatus(statusable.getStatus());
+ if (statusable.isDone()) {
+ Pageable<Path> pageable = statusable.get();
+ if (pageable != null) {
+ pcapStatus.setPageTotal(pageable.getSize());
+ }
+ }
+ return pcapStatus;
+ }
+
protected PcapStatus jobStatusToPcapStatus(JobStatus jobStatus) {
PcapStatus pcapStatus = new PcapStatus();
pcapStatus.setJobId(jobStatus.getJobId());
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 5fd9d72..866109e 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -72,6 +72,7 @@ user:
settings:
table: user_settings
cf: cf
+ job.limit: 1
pcap:
base.path: /apps/metron/pcap/input
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
index 25956e4..5d30e72 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.metron.common.Constants;
import org.apache.metron.job.JobStatus;
import org.apache.metron.job.Pageable;
+import org.apache.metron.job.manager.InMemoryJobManager;
import org.apache.metron.pcap.PcapHelper;
import org.apache.metron.pcap.PcapPages;
import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
@@ -127,6 +128,8 @@ public class PcapControllerIntegrationTest {
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ InMemoryJobManager jobManager = (InMemoryJobManager) wac.getBean("jobManager");
+ jobManager.clear();
}
@Test
@@ -216,6 +219,25 @@ public class PcapControllerIntegrationTest {
}
@Test
+ public void testTooManyJobs() throws Exception {
+ MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
+
+ mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING));
+
+ this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.jobId").value("jobId"))
+ .andExpect(jsonPath("$.jobStatus").value("RUNNING"));
+
+ this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+ .andExpect(status().isInternalServerError())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.message").value("Cannot submit job because a job is already running. Please contact the administrator to cancel job(s) with id(s) jobId"));
+
+ }
+
+ @Test
public void testGetStatus() throws Exception {
MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
@@ -265,6 +287,30 @@ public class PcapControllerIntegrationTest {
}
@Test
+ public void testGetStatusList() throws Exception {
+ MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
+
+ mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING));
+
+ this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.jobId").value("jobId"))
+ .andExpect(jsonPath("$.jobStatus").value("RUNNING"));
+
+ this.mockMvc.perform(get(pcapUrl + "?state=RUNNING").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$[0].jobId").value("jobId"))
+ .andExpect(jsonPath("$[0].jobStatus").value("RUNNING"));
+
+ this.mockMvc.perform(get(pcapUrl + "?state=SUCCEEDED").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(content().json("[]"));
+ }
+
+ @Test
public void testKillJob() throws Exception {
MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
index e9f8f9f..f99ab93 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
@@ -28,6 +28,7 @@ import org.apache.metron.job.JobException;
import org.apache.metron.job.JobNotFoundException;
import org.apache.metron.job.JobStatus;
import org.apache.metron.job.Pageable;
+import org.apache.metron.job.Statusable;
import org.apache.metron.job.manager.InMemoryJobManager;
import org.apache.metron.job.manager.JobManager;
import org.apache.metron.pcap.PcapHelper;
@@ -58,6 +59,8 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -189,6 +192,7 @@ public class PcapServiceImplTest {
when(environment.getProperty(MetronRestConstants.PCAP_FINAL_OUTPUT_PATH_SPRING_PROPERTY)).thenReturn("/final/output/path");
when(environment.getProperty(MetronRestConstants.PCAP_PAGE_SIZE_SPRING_PROPERTY)).thenReturn("100");
when(environment.getProperty(MetronRestConstants.PCAP_PDML_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/path/to/pdml/script");
+ when(environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1)).thenReturn(1);
}
@Test
@@ -334,6 +338,26 @@ public class PcapServiceImplTest {
}
@Test
+ public void submitShouldThrowExceptionOnRunningJobFound() throws Exception {
+ exception.expect(RestException.class);
+ exception.expectMessage("Cannot submit job because a job is already running. Please contact the administrator to cancel job(s) with id(s) jobId");
+
+ PcapStatus runningStatus1 = new PcapStatus();
+ runningStatus1.setJobStatus("RUNNING");
+ runningStatus1.setJobId("jobId1");
+ PcapStatus runningStatus2 = new PcapStatus();
+ runningStatus2.setJobStatus("RUNNING");
+ runningStatus2.setJobId("jobId2");
+
+ PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, new InMemoryJobManager<>(), pcapToPdmlScriptWrapper));
+ doReturn(Arrays.asList(runningStatus1, runningStatus2)).when(pcapService).getJobStatus("user", JobStatus.State.RUNNING);
+ when(environment.getProperty(MetronRestConstants.USER_JOB_LIMIT_SPRING_PROPERTY, Integer.class, 1)).thenReturn(2);
+
+ pcapService.submit("user", new FixedPcapRequest());
+ }
+
+
+ @Test
public void fixedShouldThrowRestException() throws Exception {
exception.expect(RestException.class);
exception.expectMessage("some job exception");
@@ -397,6 +421,50 @@ public class PcapServiceImplTest {
}
@Test
+ public void getStatusForStateShouldProperlyReturnJobs() throws Exception {
+ MockPcapJob mockPcapJob = mock(MockPcapJob.class);
+ JobManager jobManager = mock(JobManager.class);
+ Statusable<Path> runningJob = mock(Statusable.class);
+ JobStatus runningStatus = mock(JobStatus.class);
+ when(runningStatus.getJobId()).thenReturn("runningJob");
+ when(runningStatus.getState()).thenReturn(JobStatus.State.RUNNING);
+ when(runningJob.getStatus()).thenReturn(runningStatus);
+
+ Statusable<Path> failedJob = mock(Statusable.class);
+ when(failedJob.getStatus()).thenThrow(new JobException("job exception"));
+
+ Statusable<Path> succeededJob = mock(Statusable.class);
+ JobStatus succeededStatus = mock(JobStatus.class);
+ when(succeededStatus.getJobId()).thenReturn("succeededJob");
+ when(succeededStatus.getState()).thenReturn(JobStatus.State.SUCCEEDED);
+ when(succeededJob.isDone()).thenReturn(true);
+ when(succeededJob.getStatus()).thenReturn(succeededStatus);
+ Pageable<Path> succeededPageable = mock(Pageable.class);
+ when(succeededPageable.getSize()).thenReturn(5);
+ when(succeededJob.get()).thenReturn(succeededPageable);
+
+ when(jobManager.getJobs("user")).thenReturn(Arrays.asList(runningJob, failedJob, succeededJob));
+
+ PcapServiceImpl pcapService = new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper);
+
+ PcapStatus expectedRunningPcapStatus = new PcapStatus();
+ expectedRunningPcapStatus.setJobId("runningJob");
+ expectedRunningPcapStatus.setJobStatus(JobStatus.State.RUNNING.name());
+ Assert.assertEquals(expectedRunningPcapStatus, pcapService.getJobStatus("user", JobStatus.State.RUNNING).get(0));
+
+ PcapStatus expectedFailedPcapStatus = new PcapStatus();
+ expectedFailedPcapStatus.setJobStatus(JobStatus.State.FAILED.name());
+ expectedFailedPcapStatus.setDescription("job exception");
+ Assert.assertEquals(expectedFailedPcapStatus, pcapService.getJobStatus("user", JobStatus.State.FAILED).get(0));
+
+ PcapStatus expectedSucceededPcapStatus = new PcapStatus();
+ expectedSucceededPcapStatus.setJobId("succeededJob");
+ expectedSucceededPcapStatus.setJobStatus(JobStatus.State.SUCCEEDED.name());
+ expectedSucceededPcapStatus.setPageTotal(5);
+ Assert.assertEquals(expectedSucceededPcapStatus, pcapService.getJobStatus("user", JobStatus.State.SUCCEEDED).get(0));
+ }
+
+ @Test
public void killJobShouldKillJobAndReportStatus() throws Exception {
MockPcapJob mockPcapJob = mock(MockPcapJob.class);
JobManager jobManager = mock(JobManager.class);
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java b/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java
index 807af4d..ba436b3 100644
--- a/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java
+++ b/metron-platform/metron-job/src/main/java/org/apache/metron/job/manager/InMemoryJobManager.java
@@ -87,4 +87,8 @@ public class InMemoryJobManager<PAGE_T> implements JobManager<PAGE_T> {
return new ArrayList<>(getUserJobs(username).values());
}
+ public void clear() {
+ jobs.clear();
+ }
+
}