You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/06/13 17:33:17 UTC
incubator-metron git commit: METRON-202 Ensure resources are closed
after use (mmiklavc via nickwallen) closes apache/incubator-metron#141
Repository: incubator-metron
Updated Branches:
refs/heads/master 9041615a7 -> 6e22dd86a
METRON-202 Ensure resources are closed after use (mmiklavc via nickwallen) closes apache/incubator-metron#141
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/6e22dd86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/6e22dd86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/6e22dd86
Branch: refs/heads/master
Commit: 6e22dd86a7d4de758a4fce6f521ccc3aeb1202dd
Parents: 9041615
Author: mmiklavc <mi...@gmail.com>
Authored: Mon Jun 13 13:32:24 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Mon Jun 13 13:32:24 2016 -0400
----------------------------------------------------------------------
.../apache/metron/common/utils/JSONUtils.java | 17 +-
.../metron/common/utils/JSONUtilsTest.java | 103 ++++++++
.../metron/dataloads/cif/HBaseTableLoad.java | 255 -------------------
.../metron/test/utils/UnitTestHelper.java | 220 ++++++++++++----
4 files changed, 290 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6e22dd86/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
index 4af9ad1..4783d9b 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
@@ -54,28 +54,35 @@ public enum JSONUtils {
public <T> T load(InputStream is, TypeReference<T> ref) throws IOException {
return _mapper.get().readValue(is, ref);
}
+
public <T> T load(String is, TypeReference<T> ref) throws IOException {
return _mapper.get().readValue(is, ref);
}
+
public <T> T load(File f, TypeReference<T> ref) throws IOException {
- return _mapper.get().readValue(new BufferedInputStream(new FileInputStream(f)), ref);
+ try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
+ return _mapper.get().readValue(is, ref);
+ }
}
+
public <T> T load(InputStream is, Class<T> clazz) throws IOException {
return _mapper.get().readValue(is, clazz);
}
public <T> T load(File f, Class<T> clazz) throws IOException {
- return _mapper.get().readValue(new BufferedInputStream(new FileInputStream(f)), clazz);
+ try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
+ return _mapper.get().readValue(is, clazz);
+ }
}
+
public <T> T load(String is, Class<T> clazz) throws IOException {
return _mapper.get().readValue(is, clazz);
}
public String toJSON(Object o, boolean pretty) throws JsonProcessingException {
- if(pretty) {
+ if (pretty) {
return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsString(o);
- }
- else {
+ } else {
return _mapper.get().writeValueAsString(o);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6e22dd86/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
new file mode 100644
index 0000000..c478de5
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.utils;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class JSONUtilsTest {
+ private static File tmpDir;
+
+ /**
+ {
+ "a" : "hello",
+ "b" : "world"
+ }
+ */
+ @Multiline
+ private static String config;
+ private static File configFile;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ tmpDir = UnitTestHelper.createTempDir(new File("target/jsonutilstest"));
+ configFile = UnitTestHelper.write(new File(tmpDir, "config.json"), config);
+ }
+
+ @Test
+ public void loads_file_with_typeref() throws Exception {
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("a", "hello");
+ put("b", "world");
+ }};
+ Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, new TypeReference<Map<String, Object>>() {
+ });
+ Assert.assertThat("config not equal", actual, equalTo(expected));
+ }
+
+ @Test
+ public void loads_file_with_map_class() throws Exception {
+ Map<String, Object> expected = new HashMap<String, Object>() {{
+ put("a", "hello");
+ put("b", "world");
+ }};
+ Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, Map.class);
+ Assert.assertThat("config not equal", actual, equalTo(expected));
+ }
+
+ @Test
+ public void loads_file_with_custom_class() throws Exception {
+ TestConfig expected = new TestConfig().setA("hello").setB("world");
+ TestConfig actual = JSONUtils.INSTANCE.load(configFile, TestConfig.class);
+ Assert.assertThat("a not equal", actual.getA(), equalTo(expected.getA()));
+ Assert.assertThat("b not equal", actual.getB(), equalTo(expected.getB()));
+ }
+
+ public static class TestConfig {
+ private String a;
+ private String b;
+
+ public String getA() {
+ return a;
+ }
+
+ public TestConfig setA(String a) {
+ this.a = a;
+ return this;
+ }
+
+ public String getB() {
+ return b;
+ }
+
+ public TestConfig setB(String b) {
+ this.b = b;
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6e22dd86/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
deleted file mode 100644
index 0cff227..0000000
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads.cif;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipInputStream;
-
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Options;
-
-import java.io.BufferedInputStream;
-
-public class HBaseTableLoad {
-
- private static final Logger LOG = Logger.getLogger(HBaseTableLoad.class);
- private static Configuration conf = null;
- private String hbaseTable = "cif_table";
- private String dirName = "./";
- private boolean usefileList = false;
- private Set<String> files;
-
- /**
- * Initialization
- */
- static {
- conf = HBaseConfiguration.create();
- }
-
- public static void main(String[] args) {
-
- HBaseTableLoad ht = new HBaseTableLoad();
-
- ht.parse(args);
- //ht.LoadDirHBase();
-
- }
-
- private void LoadDirHBase() {
- LOG.info("Working on:" + dirName);
- File folder = new File(dirName);
- File[] listOfFiles = folder.listFiles();
- InputStream input;
-
- for (int i = 0; i < listOfFiles.length; i++) {
- File file = listOfFiles[i];
-
- if (file.isFile()) {
-
- // Check if filename is present in FileList
- if (usefileList)
- if (!files.contains(file.getAbsolutePath()))
- continue;
-
- // e.g. folder name is infrastructure_botnet. Col Qualifier is
- // botnet and col_family is infrastructure
-
- String col_family = folder.getName().split("_")[0];
- String col_qualifier = folder.getName().split("_")[1];
-
- // Open file
- try {
- if (file.getName().endsWith(".gz"))
- input = new BufferedInputStream(new GZIPInputStream(
- new FileInputStream(file)));
- else if (file.getName().endsWith(".zip"))
- input = new BufferedInputStream(new ZipInputStream(
- new FileInputStream(file)));
- else if (file.getName().endsWith(".json"))
- input = new BufferedInputStream((new FileInputStream(
- file)));
- else
- continue;
-
- LOG.info("Begin Loading File:" + file.getAbsolutePath());
-
- HBaseBulkPut(input, col_family, col_qualifier);
- LOG.info("Completed Loading File:" + file.getAbsolutePath());
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (ParseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- } else if (file.isDirectory()) // if sub-directory then call the
- // function recursively
- this.LoadDirHBase(file.getAbsolutePath());
- }
- }
-
- private void LoadDirHBase(String dirname) {
-
- this.dirName = dirname;
- this.LoadDirHBase();
-
- }
-
- /**
- * @param input
- * @param hbaseTable
- * @param col_family
- * @throws IOException
- * @throws ParseException
- *
- *
- * Inserts all json records picked up from the inputStream
- */
- private void HBaseBulkPut(InputStream input, String col_family,
- String col_qualifier) throws IOException, ParseException {
-
- HTable table = new HTable(conf, hbaseTable);
- JSONParser parser = new JSONParser();
-
- BufferedReader br = new BufferedReader(new InputStreamReader(input));
- String jsonString;
- List<Put> allputs = new ArrayList<Put>();
- Map json;
-
- while ((jsonString = br.readLine()) != null) {
-
- try {
-
- json = (Map) parser.parse(jsonString);
- } catch (ParseException e) {
- // System.out.println("Unable to Parse: " +jsonString);
- continue;
- }
- // Iterator iter = json.entrySet().iterator();
-
- // Get Address - either IP/domain or email and make that the Key
- Put put = new Put(Bytes.toBytes((String) json.get("address")));
-
- // We are just adding a "Y" flag to mark this address
- put.add(Bytes.toBytes(col_family), Bytes.toBytes(col_qualifier),
- Bytes.toBytes("Y"));
-
- allputs.add(put);
- }
- table.put(allputs);
- table.close();
- }
-
- private void printUsage() {
- System.out
- .println("Usage: java -cp JarFile org.apache.metron.dataloads.cif.HBaseTableLoad -d <directory> -t <tablename> -f <optional file-list>");
- }
-
- private void parse(String[] args) {
- CommandLineParser parser = new BasicParser();
- Options options = new Options();
-
- options.addOption("d", true, "description");
- options.addOption("t", true, "description");
- options.addOption("f", false, "description");
-
- CommandLine cmd = null;
- try {
- cmd = parser.parse(options, args);
-
- if (cmd.hasOption("d"))
- {
- this.dirName = cmd.getOptionValue("d");
- LOG.info("Directory Name:" + cmd.getOptionValue("d"));
- }
- else {
- LOG.info("Missing Directory Name");
- printUsage();
- System.exit(-1);
- }
-
- if (cmd.hasOption("t"))
- {
- this.hbaseTable = cmd.getOptionValue("t");
- LOG.info("HBase Table Name:" + cmd.getOptionValue("t"));
- }
- else {
- LOG.info("Missing Table Name");
- printUsage();
- System.exit(-1);
- }
-
- if (cmd.hasOption("f")) {
- this.usefileList = true;
- files = LoadFileList(cmd.getOptionValue("f"));
- LOG.info("FileList:" + cmd.getOptionValue("f"));
- }
-
- } catch (org.apache.commons.cli.ParseException e) {
- LOG.error("Failed to parse comand line properties", e);
- e.printStackTrace();
- System.exit(-1);
- }
- }
-
- private Set<String> LoadFileList(String filename) {
-
- Set<String> output = null;
- BufferedReader reader;
-
- try {
- reader = new BufferedReader(new InputStreamReader(
- new FileInputStream(filename)));
- output = new HashSet<String>();
- String in = "";
-
- while ((in = reader.readLine()) != null)
- output.add(in);
-
- reader.close();
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- return output;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6e22dd86/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
index aff48aa..a1bdaf3 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
@@ -24,61 +24,191 @@ import org.apache.log4j.PatternLayout;
import org.junit.Assert;
import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.Set;
import java.util.Stack;
+import static java.lang.String.format;
+
public class UnitTestHelper {
- public static String findDir(String name) {
- return findDir(new File("."), name);
- }
+ public static String findDir(String name) {
+ return findDir(new File("."), name);
+ }
- public static String findDir(File startDir, String name) {
- Stack<File> s = new Stack<File>();
- s.push(startDir);
- while(!s.empty()) {
- File parent = s.pop();
- if(parent.getName().equalsIgnoreCase(name)) {
- return parent.getAbsolutePath();
- }
- else {
- File[] children = parent.listFiles();
- if(children != null) {
- for (File child : children) {
- s.push(child);
- }
- }
- }
+ public static String findDir(File startDir, String name) {
+ Stack<File> s = new Stack<File>();
+ s.push(startDir);
+ while (!s.empty()) {
+ File parent = s.pop();
+ if (parent.getName().equalsIgnoreCase(name)) {
+ return parent.getAbsolutePath();
+ } else {
+ File[] children = parent.listFiles();
+ if (children != null) {
+ for (File child : children) {
+ s.push(child);
+ }
}
- return null;
+ }
}
+ return null;
+ }
- public static <T> void assertSetEqual(String type, Set<T> expectedPcapIds, Set<T> found) {
- boolean mismatch = false;
- for(T f : found) {
- if(!expectedPcapIds.contains(f)) {
- mismatch = true;
- System.out.println("Found " + type + " that I did not expect: " + f);
- }
- }
- for(T expectedId : expectedPcapIds) {
- if(!found.contains(expectedId)) {
- mismatch = true;
- System.out.println("Expected " + type + " that I did not index: " + expectedId);
- }
- }
- Assert.assertFalse(mismatch);
+ public static <T> void assertSetEqual(String type, Set<T> expectedPcapIds, Set<T> found) {
+ boolean mismatch = false;
+ for (T f : found) {
+ if (!expectedPcapIds.contains(f)) {
+ mismatch = true;
+ System.out.println("Found " + type + " that I did not expect: " + f);
+ }
+ }
+ for (T expectedId : expectedPcapIds) {
+ if (!found.contains(expectedId)) {
+ mismatch = true;
+ System.out.println("Expected " + type + " that I did not index: " + expectedId);
+ }
}
+ Assert.assertFalse(mismatch);
+ }
- public static void verboseLogging() {
- verboseLogging("%d [%p|%c|%C{1}] %m%n", Level.ALL);
+ public static void verboseLogging() {
+ verboseLogging("%d [%p|%c|%C{1}] %m%n", Level.ALL);
+ }
+
+ public static void verboseLogging(String pattern, Level level) {
+ ConsoleAppender console = new ConsoleAppender(); //create appender
+ //configure the appender
+ console.setLayout(new PatternLayout(pattern));
+ console.setThreshold(level);
+ console.activateOptions();
+ //add appender to any Logger (here is root)
+ Logger.getRootLogger().addAppender(console);
+ }
+
+ /**
+ * Create directory that is automatically cleaned up after the
+ * JVM shuts down through use of a Runtime shutdown hook.
+ *
+ * @param dir Directory to create, including missing parent directories
+ * @return File handle to the created temp directory
+ */
+ public static File createTempDir(File dir) throws IOException {
+ return createTempDir(dir, true);
+ }
+
+ /**
+ * Create directory that is automatically cleaned up after the
+ * JVM shuts down through use of a Runtime shutdown hook.
+ *
+ * @param dir Directory to create, including missing parent directories
+ * @param cleanup true/false
+ * @return File handle to the created temp directory
+ */
+ public static File createTempDir(File dir, boolean cleanup) throws IOException {
+ if (!dir.mkdirs() && !dir.exists()) {
+ throw new IOException(String.format("Failed to create directory structure '%s'", dir.toString()));
}
- public static void verboseLogging(String pattern, Level level) {
- ConsoleAppender console = new ConsoleAppender(); //create appender
- //configure the appender
- console.setLayout(new PatternLayout(pattern));
- console.setThreshold(level);
- console.activateOptions();
- //add appender to any Logger (here is root)
- Logger.getRootLogger().addAppender(console);
+ if (cleanup) {
+ addCleanupHook(dir.toPath());
}
+ return dir;
+ }
+
+ /**
+ * Create directory that is automatically cleaned up after the
+ * JVM shuts down through use of a Runtime shutdown hook.
+ *
+ * @param prefix Prefix to apply to temp directory name
+ * @return File handle to the created temp directory
+ * @throws IOException Unable to create temp directory
+ */
+ public static File createTempDir(String prefix) throws IOException {
+ return createTempDir(prefix, true);
+ }
+
+ /**
+ * Create directory that is optionally cleaned up after the
+ * JVM shuts down through use of a Runtime shutdown hook.
+ *
+ * @param prefix Prefix to apply to temp directory name
+ * @param cleanup true/false
+ * @return File handle to the created temp directory
+ * @throws IOException Unable to create temp directory
+ */
+ public static File createTempDir(String prefix, boolean cleanup) throws IOException {
+ Path tmpDir = Files.createTempDirectory(prefix);
+ addCleanupHook(tmpDir);
+ return tmpDir.toFile();
+ }
+
+ /**
+ * Adds JVM shutdown hook that will recursively delete the passed directory
+ *
+ * @param dir Directory that will be cleaned up upon JVM shutdown
+ */
+ public static void addCleanupHook(final Path dir) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ cleanDir(dir);
+ } catch (IOException e) {
+ System.out.println(format("Warning: Unable to clean folder '%s'", dir.toString()));
+ }
+ }
+ });
+ }
+
+ /**
+ * Recursive directory delete
+ *
+ * @param dir Directory to delete
+ * @throws IOException Unable to delete
+ */
+ public static void cleanDir(Path dir) throws IOException {
+ Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ if (exc == null) {
+ return FileVisitResult.CONTINUE;
+ } else {
+ throw exc;
+ }
+ }
+ });
+ Files.delete(dir);
+ }
+
+ /**
+ * Write contents to a file
+ *
+ * @param file
+ * @param contents
+ * @return file handle
+ * @throws IOException
+ */
+ public static File write(File file, String contents) throws IOException {
+ com.google.common.io.Files.createParentDirs(file);
+ com.google.common.io.Files.write(contents, file, StandardCharsets.UTF_8);
+ return file;
+ }
}