You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/08/17 15:34:55 UTC
[40/51] [abbrv] metron git commit: METRON-1722 PcapCLI should print
progress to stdout (merrimanr) closes apache/metron#1138
METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc
Branch: refs/heads/master
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr <me...@gmail.com>
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Fri Aug 10 08:25:13 2018 -0500
----------------------------------------------------------------------
metron-platform/metron-pcap-backend/README.md | 2 ++
.../java/org/apache/metron/pcap/query/CliParser.java | 4 ++++
.../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +++++++++---
.../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +++++++++
.../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++-
.../main/java/org/apache/metron/pcap/mr/PcapJob.java | 11 +++++++++++
6 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/README.md b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
-nr,--num_reducers <arg> The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol <arg> IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
-nr,--num_reducers <arg> The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query <arg> Query string to use as a filter
-st,--start_time <arg> (required) Packet start time range.
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true, String.format("Number of records to include in each output pcap file (defaults to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use for parsing start_time and end_time. Default is to use time in millis since the epoch."));
+ options.addOption(newOption("ps", "print_status", false, "Print the status of the job as it runs"));
return options;
}
@@ -125,6 +126,9 @@ public class CliParser {
//no-op
}
}
+ if (commandLine.hasOption("print_status")) {
+ config.setPrintJobStatus(true);
+ }
}
public void printHelp(String msg, Options opts) {
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
--- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
- "-records_per_file", "1000"
+ "-records_per_file", "1000",
+ "-ps"
};
Map<String, String> query = new HashMap<String, String>() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -165,6 +166,7 @@ public class PcapCliTest {
PcapOptions.START_TIME_MS.put(config, 500L);
PcapOptions.END_TIME_MS.put(config, 1000L);
PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+ PcapOptions.PRINT_JOB_STATUS.put(config, true);
when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner);
@@ -189,7 +191,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
- "-records_per_file", "1000"
+ "-records_per_file", "1000",
+ "-ps"
};
Map<String, String> query = new HashMap<String, String>() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -211,6 +214,7 @@ public class PcapCliTest {
PcapOptions.START_TIME_MS.put(config, startAsNanos / 1000000L); // needed bc defaults in config
PcapOptions.END_TIME_MS.put(config, endAsNanos / 1000000L); // needed bc defaults in config
PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+ PcapOptions.PRINT_JOB_STATUS.put(config, true);
when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner);
@@ -262,7 +266,8 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
- "-records_per_file", "1000"
+ "-records_per_file", "1000",
+ "-ps"
};
String query = "some query string";
@@ -274,6 +279,7 @@ public class PcapCliTest {
PcapOptions.START_TIME_MS.put(config, 500L); // needed bc defaults in config
PcapOptions.END_TIME_MS.put(config, 1000L); // needed bc defaults in config
PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+ PcapOptions.PRINT_JOB_STATUS.put(config, true);
when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner);
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
index 26509be..cbb8170 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
@@ -40,6 +40,7 @@ public class PcapConfig extends AbstractMapDecorator<String, Object>{
public PcapConfig(PrefixStrategy prefixStrategy) {
this();
setShowHelp(false);
+ setPrintJobStatus(false);
setBasePath("");
setBaseInterimResultPath("");
setStartTimeMs(-1L);
@@ -73,6 +74,14 @@ public class PcapConfig extends AbstractMapDecorator<String, Object>{
this.showHelp = showHelp;
}
+ public boolean printJobStatus() {
+ return PcapOptions.PRINT_JOB_STATUS.get(this, Boolean.class);
+ }
+
+ public void setPrintJobStatus(boolean printJobStatus) {
+ PcapOptions.PRINT_JOB_STATUS.put(this, printJobStatus);
+ }
+
public String getBasePath() {
return PcapOptions.BASE_PATH.get(this, String.class);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
index 3d7c4f6..203c800 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
@@ -40,7 +40,8 @@ public enum PcapOptions implements ConfigOption {
FIELDS("fields"),
FILTER_IMPL("filterImpl"),
HADOOP_CONF("hadoopConf"),
- FILESYSTEM("fileSystem");
+ FILESYSTEM("fileSystem"),
+ PRINT_JOB_STATUS("printJobStatus");
public static final BiFunction<String, Object, Path> STRING_TO_PATH =
(s, o) -> o == null ? null : new Path(o.toString());
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
index 23bd510..10f31b4 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
@@ -467,6 +467,13 @@ public class PcapJob<T> implements Statusable<Path> {
*/
@Override
public Pageable<Path> get() throws JobException, InterruptedException {
+ if (PcapOptions.PRINT_JOB_STATUS.get(configuration, Boolean.class)) {
+ try {
+ mrJob.monitorAndPrintJob();
+ } catch (IOException e) {
+ throw new JobException("Could not monitor job status", e);
+ }
+ }
for (; ; ) {
JobStatus status = getStatus();
if (status.getState() == State.SUCCEEDED
@@ -478,6 +485,10 @@ public class PcapJob<T> implements Statusable<Path> {
}
}
+ public void monitorJob() throws IOException, InterruptedException {
+ mrJob.monitorAndPrintJob();
+ }
+
private synchronized Pageable<Path> getFinalResults() {
return new PcapPages(finalResults);
}