You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/31 19:20:00 UTC
[04/50] [abbrv] metron git commit: METRON-1693: Fix Pcap CLI local FS
finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129
Parents: 9d4842f
Author: mmiklavc <mi...@gmail.com>
Authored: Wed Jul 25 08:26:45 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Jul 25 08:26:45 2018 -0600
----------------------------------------------------------------------
.../java/org/apache/metron/pcap/query/PcapCli.java | 1 -
.../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ++++++++++++---
.../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++-
.../metron/pcap/finalizer/PcapRestFinalizer.java | 10 ++++++++++
.../metron/pcap/writer/PcapResultsWriter.java | 16 ++++++++++++++++
5 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 1a23740..0b06b0c 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,7 +87,6 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
- PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" + execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
index c379515..c912e58 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
@@ -18,9 +18,13 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to local FS.
@@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer {
* as a formatted timestamp + uuid. A final sample format will look as follows:
* /base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
*/
- private static final String PCAP_CLI_FILENAME_FORMAT = "%s/pcap-data-%s+%04d.pcap";
+ private static final String PCAP_CLI_FILENAME_FORMAT = "pcap-data-%s+%04d.pcap";
+
+ @Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig,
+ List<byte[]> data, Path outputPath) throws IOException {
+ resultsWriter.writeLocal(data, outputPath.toString());
+ }
@Override
protected Path getOutputPath(Map<String, Object> config, int partition) {
- Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config, PcapOptions.STRING_TO_PATH, Path.class);
String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config, String.class);
- return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath, prefix, partition));
+ return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix, partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
index 2c55e15..8dcc401 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
@@ -81,7 +81,7 @@ public abstract class PcapFinalizer implements Finalizer<Path> {
for (List<byte[]> data : partitions) {
Path outputPath = getOutputPath(config, part++);
if (data.size() > 0) {
- getResultsWriter().write(hadoopConfig, data, outputPath.toUri().getPath());
+ write(resultsWriter, hadoopConfig, data, outputPath);
outFiles.add(outputPath);
}
}
@@ -100,6 +100,8 @@ public abstract class PcapFinalizer implements Finalizer<Path> {
return new PcapPages(outFiles);
}
+ protected abstract void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig, List<byte[]> data, Path outputPath) throws IOException;
+
protected abstract Path getOutputPath(Map<String, Object> config, int partition);
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
index 95907df..93a3222 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
@@ -18,11 +18,15 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.config.PcapOptions;
import java.util.Map;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to HDFS.
@@ -34,6 +38,12 @@ public class PcapRestFinalizer extends PcapFinalizer {
private String jobType = Statusable.JobType.MAP_REDUCE.name();
@Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration hadoopConfig,
+ List<byte[]> data, Path outputPath) throws IOException {
+ resultsWriter.write(hadoopConfig, data, outputPath.toString());
+ }
+
+ @Override
protected Path getOutputPath(Map<String, Object> config, int partition) {
String finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config, String.class);
String user = PcapOptions.USERNAME.get(config, String.class);
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java
index 62ac27c..05d9991 100644
--- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java
+++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/writer/PcapResultsWriter.java
@@ -18,6 +18,8 @@
package org.apache.metron.pcap.writer;
import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +41,20 @@ public class PcapResultsWriter {
}
/**
+ * Write out pcaps to the local FS
+ *
+ * @param pcaps pcap data to write. Pre-merged format as a list of pcaps as byte arrays.
+ * @param outPath where to write the pcap data to.
+ * @throws IOException I/O issue encountered.
+ */
+ public void writeLocal(List<byte[]> pcaps, String outPath) throws IOException {
+ File out = new File(outPath);
+ try (FileOutputStream fos = new FileOutputStream(out)) {
+ fos.write(mergePcaps(pcaps));
+ }
+ }
+
+ /**
* Creates a pcap file with proper global header from individual pcaps.
*
* @param pcaps pcap records to merge into a pcap file with header.