You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/01/26 15:18:46 UTC
[68/89] [abbrv] incubator-metron git commit: Move all com/apache
folders to org/apache
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractConfigTest.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractConfigTest.java
new file mode 100644
index 0000000..ba9b03d
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractConfigTest.java
@@ -0,0 +1,299 @@
+
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.apache.metron.test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.apache.metron.helpers.topology.SettingsLoader;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractConfigTest</code> is
+ * an abstract base class for implementing JUnit tests that need to use
+ * config to connect to ZooKeeper and HBase. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Oct 10, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractConfigTest extends AbstractTestContext{
+ /**
+ * The configPath.
+ */
+ protected String configPath=null;
+
+ /**
+ * The configName.
+ */
+ protected String configName=null;
+
+ /**
+ * The config.
+ */
+ private Configuration config=null;
+
+ /**
+ * The settings.
+ */
+ Map<String, String> settings=null;
+
+ /**
+ * The schemaJsonString.
+ */
+ private String schemaJsonString = null;
+ /**
+ * Any Object for mavenMode
+ * @parameter
+ * expression="${mode}"
+ * default-value="local"
+ */
+ private Object mode="local";
+
+ /**
+ * Constructs a new <code>AbstractConfigTest</code> instance.
+ * @throws Exception
+ */
+ public AbstractConfigTest() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Constructs a new <code>AbstractTestContext</code> instance.
+ * @param name the name of the test case.
+ */
+ public AbstractConfigTest(String name) {
+ super(name);
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+ protected void setUp(String configName) throws Exception {
+ super.setUp();
+ this.setConfigPath("src/test/resources/config/"+getClass().getSimpleName()+".config");
+ try {
+ this.setConfig(new PropertiesConfiguration(this.getConfigPath()));
+
+ Map configOptions= SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config, configName+"=");
+ this.setSettings(SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config, configName + "."));
+ this.getSettings().put(configName, (String) configOptions.get(configName));
+ } catch (ConfigurationException e) {
+ fail("Config not found !!"+e);
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+
+ }
+
+
+ /**
+ * validateJsonData
+ * @param jsonSchema
+ * @param jsonData
+ * @return
+ * @throws Exception
+ */
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+ throws Exception {
+
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+ System.out.println(report);
+
+ return report.toString().contains("success");
+ }
+
+ protected String readSchemaFromFile(URL schema_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ schema_url.getFile()));
+ String line;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ sb.append(line);
+ }
+ br.close();
+
+ String schema_string = sb.toString().replaceAll("\n", "");
+ schema_string = schema_string.replaceAll(" ", "");
+
+ System.out.println("Read in schema: " + schema_string);
+
+ return schema_string;
+ }
+
+ protected String[] readTestDataFromFile(String test_data_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ new File(test_data_url)));
+ ArrayList<String> inputDataLines = new ArrayList<String>();
+
+ String line;
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ inputDataLines.add(line.toString().replaceAll("\n", ""));
+ }
+ br.close();
+ String[] inputData = new String[inputDataLines.size()];
+ inputData = inputDataLines.toArray(inputData);
+
+ return inputData;
+ }
+ /**
+ * Skip Tests
+ */
+ public boolean skipTests(Object mode){
+ if(mode.toString().equals("local")){
+ return true;
+ }else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the mode.
+ * @return the mode.
+ */
+
+ public Object getMode() {
+ return mode;
+ }
+
+ /**
+ * Sets the mode.
+ * @param mode the mode.
+ */
+
+ public void setMode(Object mode) {
+
+ this.mode = mode;
+ }
+
+
+ /**
+ * @param readSchemaFromFile
+ */
+ public void setSchemaJsonString(String schemaJsonString) {
+ this.schemaJsonString=schemaJsonString;
+ }
+
+
+ /**
+ * @return
+ */
+ public String getSchemaJsonString() {
+ return this.schemaJsonString;
+ }
+
+ /**
+ * Returns the configPath.
+ * @return the configPath.
+ */
+ public String getConfigPath() {
+ return configPath;
+ }
+
+ /**
+ * Sets the configPath.
+ * @param configPath the configPath.
+ */
+ public void setConfigPath(String configPath) {
+ this.configPath = configPath;
+ }
+ /**
+ * Returns the config.
+ * @return the config.
+ */
+
+ public Configuration getConfig() {
+ return config;
+ }
+
+ /**
+ * Sets the config.
+ * @param config the config.
+ */
+
+ public void setConfig(Configuration config) {
+
+ this.config = config;
+ }
+ /**
+ * Returns the settings.
+ * @return the settings.
+ */
+
+ public Map<String, String> getSettings() {
+ return settings;
+ }
+
+ /**
+ * Sets the settings.
+ * @param settings the settings.
+ */
+
+ public void setSettings(Map<String, String> settings) {
+ this.settings = settings;
+ }
+ /**
+ * Returns the configName.
+ * @return the configName.
+ */
+ public String getConfigName() {
+ return configName;
+ }
+
+ /**
+ * Sets the configName.
+ * @param configName the configName.
+ */
+ public void setConfigName(String configName) {
+ this.configName = configName;
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractSchemaTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractSchemaTest.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractSchemaTest.java
new file mode 100644
index 0000000..87ce851
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractSchemaTest.java
@@ -0,0 +1,198 @@
+
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.apache.metron.test;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.net.URL;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractSchemaTest</code> is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * Json Schema. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Aug 7, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractSchemaTest extends AbstractConfigTest{
+
+
+ /**
+ * The schemaJsonString.
+ */
+ private String schemaJsonString = null;
+ /**
+ * Any Object for mavenMode
+ * @parameter
+ * expression="${mode}"
+ * default-value="local"
+ */
+ private Object mode="local";
+
+ /**
+ * Constructs a new <code>AbstractTestContext</code> instance.
+ * @throws Exception
+ */
+ public AbstractSchemaTest() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Constructs a new <code>AbstractTestContext</code> instance.
+ * @param name the name of the test case.
+ */
+ public AbstractSchemaTest(String name) {
+ super(name);
+ try{
+ if(System.getProperty("mode")!=null){
+ setMode(System.getProperty("mode") );
+ }else
+ {
+ setMode("local");
+ }
+ }catch(Exception ex){
+ setMode("local");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+
+ }
+
+
+ /**
+ * validateJsonData
+ * @param jsonSchema
+ * @param jsonData
+ * @return
+ * @throws Exception
+ */
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+ throws Exception {
+
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+ System.out.println(report);
+
+ return report.toString().contains("success");
+ }
+
+ protected String readSchemaFromFile(URL schema_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ schema_url.getFile()));
+ String line;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ sb.append(line);
+ }
+ br.close();
+
+ String schema_string = sb.toString().replaceAll("\n", "");
+ schema_string = schema_string.replaceAll(" ", "");
+
+ System.out.println("Read in schema: " + schema_string);
+
+ return schema_string;
+
+ }
+
+ /**
+ * Skip Tests
+ */
+ public boolean skipTests(Object mode){
+ if(mode.toString().equals("local")){
+ return true;
+ }else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the mode.
+ * @return the mode.
+ */
+
+ public Object getMode() {
+ return mode;
+ }
+
+ /**
+ * Sets the mode.
+ * @param mode the mode.
+ */
+
+ public void setMode(Object mode) {
+
+ this.mode = mode;
+ }
+
+
+ /**
+
+ * @param readSchemaFromFile
+ */
+
+ public void setSchemaJsonString(String schemaJsonString) {
+ this.schemaJsonString=schemaJsonString;
+ }
+
+
+ /**
+
+ * @return
+ */
+
+ public String getSchemaJsonString() {
+ return this.schemaJsonString;
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractTestContext.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractTestContext.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractTestContext.java
new file mode 100644
index 0000000..1e30698
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/test/AbstractTestContext.java
@@ -0,0 +1,189 @@
+
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.apache.metron.test;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractTestContext</code> is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * test properties. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Aug 7, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractTestContext extends TestCase{
+ /**
+ * The testProps.
+ */
+ protected File testPropFile=null;
+
+ /**
+ * The properties loaded for test.
+ */
+ protected Properties testProperties=new Properties();
+
+ /**
+ * Any Object for mavenMode
+ * @parameter
+ * expression="${mode}"
+ * default-value="global"
+ */
+ private Object mode="local";
+
+ /**
+ * Constructs a new <code>AbstractTestContext</code> instance.
+ */
+ public AbstractTestContext() {
+ super();
+ }
+
+ /**
+ * Constructs a new <code>AbstractTestContext</code> instance.
+ * @param name the name of the test case.
+ */
+ public AbstractTestContext(String name) {
+ super(name);
+ try{
+ if(System.getProperty("mode")!=null){
+ setMode(System.getProperty("mode") );
+ }else
+ {
+ setMode("local");
+ }
+ }catch(Exception ex){
+ setMode("local");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+ @Override
+ protected void setUp() throws Exception {
+ InputStream input=null;
+ File directory = new File("src/test/resources");
+ if (!directory.isDirectory()) {
+ return;
+ }
+ File file = new File(directory, getClass().getSimpleName() + ".properties");
+ if (!file.canRead()) {
+ return;
+ }
+ setTestPropFile(file);
+ try{
+ input=new FileInputStream(file);
+ testProperties.load(input);
+ }catch(IOException ex){
+ ex.printStackTrace();
+ fail("failed to load properties");
+ }
+
+
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+
+ }
+
+ /**
+ * Returns the testProperties.
+ * @return the testProperties.
+ */
+
+ public Properties getTestProperties() {
+ return testProperties;
+ }
+
+ /**
+ * Sets the testProperties.
+ * @param testProperties the testProperties.
+ */
+
+ public void setTestProperties(Properties testProperties) {
+
+ this.testProperties = testProperties;
+ }
+ /**
+ * Returns the testPropFile.
+ * @return the testPropFile.
+ */
+
+ public File getTestPropFile() {
+ return testPropFile;
+ }
+
+ /**
+ * Sets the testPropFile.
+ * @param testPropFile the testPropFile.
+ */
+
+ public void setTestPropFile(File testPropFile) {
+
+ this.testPropFile = testPropFile;
+ }
+
+ /**
+ * Skip Tests
+ */
+ public boolean skipTests(Object mode){
+ if(mode.toString().equals("local")){
+ return true;
+ }else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the mode.
+ * @return the mode.
+ */
+
+ public Object getMode() {
+ return mode;
+ }
+
+ /**
+ * Sets the mode.
+ * @param mode the mode.
+ */
+
+ public void setMode(Object mode) {
+
+ this.mode = mode;
+ }
+
+ }
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/BasicTldExtractor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/BasicTldExtractor.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/BasicTldExtractor.java
new file mode 100644
index 0000000..dc4c447
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/BasicTldExtractor.java
@@ -0,0 +1,137 @@
+package com.apache.metron.tldextractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class BasicTldExtractor implements Serializable {
+ private static final long serialVersionUID = -7440226111118873815L;
+ private StringBuilder sb = new StringBuilder();
+
+ private Pattern pattern;
+
+ /**
+ * The inputFile.
+ */
+ private String inputFile ="effective_tld_names.dat";
+
+ public BasicTldExtractor(String filePath) {
+ this.inputFile=filePath;
+ this.init();
+ }
+
+ public BasicTldExtractor() {
+ this.init();
+ }
+
+ private void init(){
+ try {
+ ArrayList<String> terms = new ArrayList<String>();
+
+
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ getClass().getClassLoader().getResourceAsStream(inputFile)));
+ String s = null;
+ while ((s = br.readLine()) != null) {
+ s = s.trim();
+ if (s.length() == 0 || s.startsWith("//") || s.startsWith("!"))
+ continue;
+ terms.add(s);
+ }
+ Collections.sort(terms, new StringLengthComparator());
+ for (String t : terms)
+ add(t);
+ compile();
+ br.close();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ protected void add(String s) {
+ s = s.replace(".", "\\.");
+ s = "\\." + s;
+ if (s.startsWith("*")) {
+ s = s.replace("*", ".+");
+ sb.append(s).append("|");
+ } else {
+ sb.append(s).append("|");
+ }
+ }
+
+ public void compile() {
+ if (sb.length() > 0)
+ sb.deleteCharAt(sb.length() - 1);
+ sb.insert(0, "[^.]+?(");
+ sb.append(")$");
+ pattern = Pattern.compile(sb.toString());
+ sb = null;
+ }
+
+ public String extract2LD(String host) {
+ Matcher m = pattern.matcher(host);
+ if (m.find()) {
+ return m.group(0);
+ }
+ return null;
+ }
+
+ public String extractTLD(String host) {
+ Matcher m = pattern.matcher(host);
+ if (m.find()) {
+ return m.group(1);
+ }
+ return null;
+ }
+
+ public static class StringLengthComparator implements Comparator<String> {
+ public int compare(String s1, String s2) {
+ if (s1.length() > s2.length())
+ return -1;
+ if (s1.length() < s2.length())
+ return 1;
+ return 0;
+ }
+ }
+ /**
+ * Returns the sb.
+ * @return the sb.
+ */
+
+ public StringBuilder getSb() {
+ return sb;
+ }
+
+ /**
+ * Sets the sb.
+ * @param sb the sb.
+ */
+
+ public void setSb(StringBuilder sb) {
+
+ this.sb = sb;
+ }
+ /**
+ * Returns the inputFile.
+ * @return the inputFile.
+ */
+
+ public String getInputFile() {
+ return inputFile;
+ }
+
+ /**
+ * Sets the inputFile.
+ * @param inputFile the inputFile.
+ */
+
+ public void setInputFile(String inputFile) {
+
+ this.inputFile = inputFile;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/test/BasicTldExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/test/BasicTldExtractorTest.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/test/BasicTldExtractorTest.java
new file mode 100644
index 0000000..936626a
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/tldextractor/test/BasicTldExtractorTest.java
@@ -0,0 +1,125 @@
+
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.apache.metron.tldextractor.test;
+
+import com.apache.metron.test.AbstractConfigTest;
+import com.apache.metron.tldextractor.BasicTldExtractor;
+
+
+ /**
+ * <ul>
+ * <li>Title: Basic TLD Extractor Test</li>
+ * <li>Description: Basic TLD Extractor class test</li>
+ * <li>Created: Feb 26, 2015</li>
+ * </ul>
+ * @author $Author: $
+ * @version $Revision: 1.1 $
+ */
+public class BasicTldExtractorTest extends AbstractConfigTest {
+ /**
+ * The tldExtractor.
+ */
+
+ private BasicTldExtractor tldExtractor=null;
+
+ /**
+ * Constructs a new <code>BasicTldExtractorTest</code> instance.
+ * @param name
+ */
+
+ public BasicTldExtractorTest(String name) {
+ super(name);
+ }
+
+ /**
+
+ * @throws java.lang.Exception
+ */
+ protected static void setUpBeforeClass() throws Exception {
+ }
+
+ /**
+
+ * @throws java.lang.Exception
+ */
+ protected static void tearDownAfterClass() throws Exception {
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+
+ protected void setUp() throws Exception {
+ super.setUp("com.apache.metron.tldextractor.test.BasicTldExtractorTest");
+ this.tldExtractor=new BasicTldExtractor(this.getConfig().getString("logFile"));
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /**
+ * Test method for {@link com.apache.metron.tldextractor.BasicTldExtractor#BasicTldExtractor()}.
+ */
+ public void testBasicTldExtractor() {
+ assertNotNull(this.tldExtractor);
+ }
+
+ /**
+ * Test method for {@link com.apache.metron.tldextractor.BasicTldExtractor#extract2LD(java.lang.String)}.
+ */
+ public void testExtract2LD() {
+ //fail("Not yet implemented");
+ }
+
+ /**
+ * Test method for {@link com.apache.metron.tldextractor.BasicTldExtractor#extractTLD(java.lang.String)}.
+ */
+ public void testExtractTLD()
+ {
+ String result = this.tldExtractor.extractTLD("cisco.com");
+ System.out.println("result ="+result);
+ }
+ /**
+ * Returns the tldExtractor.
+ * @return the tldExtractor.
+ */
+
+ public BasicTldExtractor getTldExtractor() {
+ return tldExtractor;
+ }
+
+ /**
+ * Sets the tldExtractor.
+ * @param tldExtractor the tldExtractor.
+ */
+
+ public void setTldExtractor(BasicTldExtractor tldExtractor) {
+
+ this.tldExtractor = tldExtractor;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/ThreatIntelLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/ThreatIntelLoader.java b/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/ThreatIntelLoader.java
deleted file mode 100644
index 3e08b30..0000000
--- a/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/ThreatIntelLoader.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package com.apache.metron.dataloads;
-
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.log4j.Logger;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONArray;
-
-import com.apache.metron.dataloads.interfaces.ThreatIntelSource;
-
-public class ThreatIntelLoader {
-
-
- private static final Logger LOG = Logger.getLogger(ThreatIntelLoader.class);
-
- private static int BULK_SIZE = 50;
-
- public static void main(String[] args) {
-
- PropertiesConfiguration sourceConfig = null;
- ThreatIntelSource threatIntelSource = null;
- ArrayList<Put> putList = null;
- HTable table = null;
- Configuration hConf = null;
-
- CommandLine commandLine = parseCommandLine(args);
- File configFile = new File(commandLine.getOptionValue("configFile"));
-
- try {
- sourceConfig = new PropertiesConfiguration(configFile);
- } catch (org.apache.commons.configuration.ConfigurationException e) {
- LOG.error("Error in configuration file " + configFile);
- LOG.error(e);
- System.exit(-1);
- }
-
- try {
- threatIntelSource = (ThreatIntelSource) Class.forName(commandLine.getOptionValue("source")).newInstance();
- threatIntelSource.initializeSource(sourceConfig);
- } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) {
- LOG.error("Error while trying to load class " + commandLine.getOptionValue("source"));
- LOG.error(e);
- System.exit(-1);
- }
-
- hConf = HBaseConfiguration.create();
- try {
- table = new HTable(hConf, commandLine.getOptionValue("table"));
- } catch (IOException e) {
- LOG.error("Exception when processing HBase config");
- LOG.error(e);
- System.exit(-1);
- }
-
-
- putList = new ArrayList<Put>();
-
- while (threatIntelSource.hasNext()) {
-
- JSONObject intel = threatIntelSource.next();
-
- /*
- * If any of the required fields from threatIntelSource are
- * missing, or contain invalid data, don't put it in HBase.
- */
- try {
-
- putList.add(putRequestFromIntel(intel));
-
- if (putList.size() == BULK_SIZE) {
- table.put(putList);
- putList.clear();
- }
-
- } catch (NullPointerException|ClassCastException e) {
- LOG.error("Exception while processing intel object");
- LOG.error(intel.toString());
- LOG.error(e);
- } catch (InterruptedIOException|org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException e) {
- LOG.error("Problem communicationg with HBase");
- LOG.error(e);
- System.exit(-1);
- }
- }
-
- }
- /*
- * Takes a JSONObject from a ThreatIntelSource implementation, ensures
- * that the format of the returned JSONObect is correct, and returns
- * a Put request for HBase.
- *
- * @param intel The JSONObject from a ThreatIntelSource
- * @return A put request for the intel data
- * @throws NullPointerException If a required field is missing
- * @throws ClassCastException If a field has an invalid type
- *
- */
- private static Put putRequestFromIntel(JSONObject intel) {
-
- Put tempPut = new Put(Bytes.toBytes((String) intel.get("indicator")));
-
- JSONArray intelArray = (JSONArray) intel.get("data");
-
- tempPut.add(Bytes.toBytes("source"),
- Bytes.toBytes((String) intel.get("source")),
- Bytes.toBytes(intelArray.toString()));
-
- return tempPut;
- }
- /*
- * Handles parsing of command line options and validates the options are used
- * correctly. This will not validate the value of the options, it will just
- * ensure that the required options are used. If the options are used
- * incorrectly, the help is printed, and the program exits.
- *
- * @param args The arguments from the CLI
- * @return A CommandLine with the CLI arguments
- *
- */
- private static CommandLine parseCommandLine(String[] args) {
-
- CommandLineParser parser = new BasicParser();
- CommandLine cli = null;
-
- Options options = new Options();
-
- options.addOption(OptionBuilder.withArgName("s").
- withLongOpt("source").
- isRequired(true).
- hasArg(true).
- withDescription("Source class to use").
- create()
- );
- options.addOption(OptionBuilder.withArgName("t").
- withLongOpt("table").
- isRequired(true).
- hasArg(true).
- withDescription("HBase table to load into").
- create()
- );
- options.addOption(OptionBuilder.withArgName("c").
- withLongOpt("configFile").
- hasArg(true).
- withDescription("Configuration file for source class").
- create()
- );
-
- try {
- cli = parser.parse(options, args);
- } catch(org.apache.commons.cli.ParseException e) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("ThreatIntelLoader", options, true);
- System.exit(-1);
- }
-
- return cli;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/cif/HBaseTableLoad.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/cif/HBaseTableLoad.java b/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/cif/HBaseTableLoad.java
deleted file mode 100644
index 9fb06b1..0000000
--- a/metron-streaming/Metron-DataLoads/src/main/java/com/apache/metron/dataloads/cif/HBaseTableLoad.java
+++ /dev/null
@@ -1,238 +0,0 @@
-package com.apache.metron.dataloads.cif;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipInputStream;
-
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Options;
-
-import java.io.BufferedInputStream;
-
-public class HBaseTableLoad {
-
- private static final Logger LOG = Logger.getLogger(HBaseTableLoad.class);
- private static Configuration conf = null;
- private String hbaseTable = "cif_table";
- private String dirName = "./";
- private boolean usefileList = false;
- private Set<String> files;
-
- /**
- * Initialization
- */
- static {
- conf = HBaseConfiguration.create();
- }
-
- public static void main(String[] args) {
-
- HBaseTableLoad ht = new HBaseTableLoad();
-
- ht.parse(args);
- //ht.LoadDirHBase();
-
- }
-
- private void LoadDirHBase() {
- LOG.info("Working on:" + dirName);
- File folder = new File(dirName);
- File[] listOfFiles = folder.listFiles();
- InputStream input;
-
- for (int i = 0; i < listOfFiles.length; i++) {
- File file = listOfFiles[i];
-
- if (file.isFile()) {
-
- // Check if filename is present in FileList
- if (usefileList)
- if (!files.contains(file.getAbsolutePath()))
- continue;
-
- // e.g. folder name is infrastructure_botnet. Col Qualifier is
- // botnet and col_family is infrastructure
-
- String col_family = folder.getName().split("_")[0];
- String col_qualifier = folder.getName().split("_")[1];
-
- // Open file
- try {
- if (file.getName().endsWith(".gz"))
- input = new BufferedInputStream(new GZIPInputStream(
- new FileInputStream(file)));
- else if (file.getName().endsWith(".zip"))
- input = new BufferedInputStream(new ZipInputStream(
- new FileInputStream(file)));
- else if (file.getName().endsWith(".json"))
- input = new BufferedInputStream((new FileInputStream(
- file)));
- else
- continue;
-
- LOG.info("Begin Loading File:" + file.getAbsolutePath());
-
- HBaseBulkPut(input, col_family, col_qualifier);
- LOG.info("Completed Loading File:" + file.getAbsolutePath());
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (ParseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- } else if (file.isDirectory()) // if sub-directory then call the
- // function recursively
- this.LoadDirHBase(file.getAbsolutePath());
- }
- }
-
- private void LoadDirHBase(String dirname) {
-
- this.dirName = dirname;
- this.LoadDirHBase();
-
- }
-
- /**
- * @param input
- * @param hbaseTable
- * @param col_family
- * @throws IOException
- * @throws ParseException
- *
- *
- * Inserts all json records picked up from the inputStream
- */
- private void HBaseBulkPut(InputStream input, String col_family,
- String col_qualifier) throws IOException, ParseException {
-
- HTable table = new HTable(conf, hbaseTable);
- JSONParser parser = new JSONParser();
-
- BufferedReader br = new BufferedReader(new InputStreamReader(input));
- String jsonString;
- List<Put> allputs = new ArrayList<Put>();
- Map json;
-
- while ((jsonString = br.readLine()) != null) {
-
- try {
-
- json = (Map) parser.parse(jsonString);
- } catch (ParseException e) {
- // System.out.println("Unable to Parse: " +jsonString);
- continue;
- }
- // Iterator iter = json.entrySet().iterator();
-
- // Get Address - either IP/domain or email and make that the Key
- Put put = new Put(Bytes.toBytes((String) json.get("address")));
-
- // We are just adding a "Y" flag to mark this address
- put.add(Bytes.toBytes(col_family), Bytes.toBytes(col_qualifier),
- Bytes.toBytes("Y"));
-
- allputs.add(put);
- }
- table.put(allputs);
- table.close();
- }
-
- private void printUsage() {
- System.out
- .println("Usage: java -cp JarFile com.apache.metron.dataloads.cif.HBaseTableLoad -d <directory> -t <tablename> -f <optional file-list>");
- }
-
- private void parse(String[] args) {
- CommandLineParser parser = new BasicParser();
- Options options = new Options();
-
- options.addOption("d", true, "description");
- options.addOption("t", true, "description");
- options.addOption("f", false, "description");
-
- CommandLine cmd = null;
- try {
- cmd = parser.parse(options, args);
-
- if (cmd.hasOption("d"))
- {
- this.dirName = cmd.getOptionValue("d");
- LOG.info("Directory Name:" + cmd.getOptionValue("d"));
- }
- else {
- LOG.info("Missing Directory Name");
- printUsage();
- System.exit(-1);
- }
-
- if (cmd.hasOption("t"))
- {
- this.hbaseTable = cmd.getOptionValue("t");
- LOG.info("HBase Table Name:" + cmd.getOptionValue("t"));
- }
- else {
- LOG.info("Missing Table Name");
- printUsage();
- System.exit(-1);
- }
-
- if (cmd.hasOption("f")) {
- this.usefileList = true;
- files = LoadFileList(cmd.getOptionValue("f"));
- LOG.info("FileList:" + cmd.getOptionValue("f"));
- }
-
- } catch (org.apache.commons.cli.ParseException e) {
- LOG.error("Failed to parse comand line properties", e);
- e.printStackTrace();
- System.exit(-1);
- }
- }
-
- private Set<String> LoadFileList(String filename) {
-
- Set<String> output = null;
- BufferedReader reader;
-
- try {
- reader = new BufferedReader(new InputStreamReader(
- new FileInputStream(filename)));
- output = new HashSet<String>();
- String in = "";
-
- while ((in = reader.readLine()) != null)
- output.add(in);
-
- reader.close();
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- return output;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelLoader.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelLoader.java
new file mode 100644
index 0000000..3e08b30
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelLoader.java
@@ -0,0 +1,174 @@
+package com.apache.metron.dataloads;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.log4j.Logger;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONArray;
+
+import com.apache.metron.dataloads.interfaces.ThreatIntelSource;
+
+public class ThreatIntelLoader {
+
+
+ private static final Logger LOG = Logger.getLogger(ThreatIntelLoader.class);
+
+ private static int BULK_SIZE = 50;
+
+ public static void main(String[] args) {
+
+ PropertiesConfiguration sourceConfig = null;
+ ThreatIntelSource threatIntelSource = null;
+ ArrayList<Put> putList = null;
+ HTable table = null;
+ Configuration hConf = null;
+
+ CommandLine commandLine = parseCommandLine(args);
+ File configFile = new File(commandLine.getOptionValue("configFile"));
+
+ try {
+ sourceConfig = new PropertiesConfiguration(configFile);
+ } catch (org.apache.commons.configuration.ConfigurationException e) {
+ LOG.error("Error in configuration file " + configFile);
+ LOG.error(e);
+ System.exit(-1);
+ }
+
+ try {
+ threatIntelSource = (ThreatIntelSource) Class.forName(commandLine.getOptionValue("source")).newInstance();
+ threatIntelSource.initializeSource(sourceConfig);
+ } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) {
+ LOG.error("Error while trying to load class " + commandLine.getOptionValue("source"));
+ LOG.error(e);
+ System.exit(-1);
+ }
+
+ hConf = HBaseConfiguration.create();
+ try {
+ table = new HTable(hConf, commandLine.getOptionValue("table"));
+ } catch (IOException e) {
+ LOG.error("Exception when processing HBase config");
+ LOG.error(e);
+ System.exit(-1);
+ }
+
+
+ putList = new ArrayList<Put>();
+
+ while (threatIntelSource.hasNext()) {
+
+ JSONObject intel = threatIntelSource.next();
+
+ /*
+ * If any of the required fields from threatIntelSource are
+ * missing, or contain invalid data, don't put it in HBase.
+ */
+ try {
+
+ putList.add(putRequestFromIntel(intel));
+
+ if (putList.size() == BULK_SIZE) {
+ table.put(putList);
+ putList.clear();
+ }
+
+ } catch (NullPointerException|ClassCastException e) {
+ LOG.error("Exception while processing intel object");
+ LOG.error(intel.toString());
+ LOG.error(e);
+ } catch (InterruptedIOException|org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException e) {
+ LOG.error("Problem communicationg with HBase");
+ LOG.error(e);
+ System.exit(-1);
+ }
+ }
+
+ }
+ /*
+ * Takes a JSONObject from a ThreatIntelSource implementation, ensures
+ * that the format of the returned JSONObect is correct, and returns
+ * a Put request for HBase.
+ *
+ * @param intel The JSONObject from a ThreatIntelSource
+ * @return A put request for the intel data
+ * @throws NullPointerException If a required field is missing
+ * @throws ClassCastException If a field has an invalid type
+ *
+ */
+ private static Put putRequestFromIntel(JSONObject intel) {
+
+ Put tempPut = new Put(Bytes.toBytes((String) intel.get("indicator")));
+
+ JSONArray intelArray = (JSONArray) intel.get("data");
+
+ tempPut.add(Bytes.toBytes("source"),
+ Bytes.toBytes((String) intel.get("source")),
+ Bytes.toBytes(intelArray.toString()));
+
+ return tempPut;
+ }
+ /*
+ * Handles parsing of command line options and validates the options are used
+ * correctly. This will not validate the value of the options, it will just
+ * ensure that the required options are used. If the options are used
+ * incorrectly, the help is printed, and the program exits.
+ *
+ * @param args The arguments from the CLI
+ * @return A CommandLine with the CLI arguments
+ *
+ */
+ private static CommandLine parseCommandLine(String[] args) {
+
+ CommandLineParser parser = new BasicParser();
+ CommandLine cli = null;
+
+ Options options = new Options();
+
+ options.addOption(OptionBuilder.withArgName("s").
+ withLongOpt("source").
+ isRequired(true).
+ hasArg(true).
+ withDescription("Source class to use").
+ create()
+ );
+ options.addOption(OptionBuilder.withArgName("t").
+ withLongOpt("table").
+ isRequired(true).
+ hasArg(true).
+ withDescription("HBase table to load into").
+ create()
+ );
+ options.addOption(OptionBuilder.withArgName("c").
+ withLongOpt("configFile").
+ hasArg(true).
+ withDescription("Configuration file for source class").
+ create()
+ );
+
+ try {
+ cli = parser.parse(options, args);
+ } catch(org.apache.commons.cli.ParseException e) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("ThreatIntelLoader", options, true);
+ System.exit(-1);
+ }
+
+ return cli;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
new file mode 100644
index 0000000..9fb06b1
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/cif/HBaseTableLoad.java
@@ -0,0 +1,238 @@
+package com.apache.metron.dataloads.cif;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipInputStream;
+
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+
+import java.io.BufferedInputStream;
+
+public class HBaseTableLoad {
+
+ private static final Logger LOG = Logger.getLogger(HBaseTableLoad.class);
+ private static Configuration conf = null;
+ private String hbaseTable = "cif_table";
+ private String dirName = "./";
+ private boolean usefileList = false;
+ private Set<String> files;
+
+ /**
+ * Initialization
+ */
+ static {
+ conf = HBaseConfiguration.create();
+ }
+
+ public static void main(String[] args) {
+
+ HBaseTableLoad ht = new HBaseTableLoad();
+
+ ht.parse(args);
+ //ht.LoadDirHBase();
+
+ }
+
+ private void LoadDirHBase() {
+ LOG.info("Working on:" + dirName);
+ File folder = new File(dirName);
+ File[] listOfFiles = folder.listFiles();
+ InputStream input;
+
+ for (int i = 0; i < listOfFiles.length; i++) {
+ File file = listOfFiles[i];
+
+ if (file.isFile()) {
+
+ // Check if filename is present in FileList
+ if (usefileList)
+ if (!files.contains(file.getAbsolutePath()))
+ continue;
+
+ // e.g. folder name is infrastructure_botnet. Col Qualifier is
+ // botnet and col_family is infrastructure
+
+ String col_family = folder.getName().split("_")[0];
+ String col_qualifier = folder.getName().split("_")[1];
+
+ // Open file
+ try {
+ if (file.getName().endsWith(".gz"))
+ input = new BufferedInputStream(new GZIPInputStream(
+ new FileInputStream(file)));
+ else if (file.getName().endsWith(".zip"))
+ input = new BufferedInputStream(new ZipInputStream(
+ new FileInputStream(file)));
+ else if (file.getName().endsWith(".json"))
+ input = new BufferedInputStream((new FileInputStream(
+ file)));
+ else
+ continue;
+
+ LOG.info("Begin Loading File:" + file.getAbsolutePath());
+
+ HBaseBulkPut(input, col_family, col_qualifier);
+ LOG.info("Completed Loading File:" + file.getAbsolutePath());
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (ParseException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ } else if (file.isDirectory()) // if sub-directory then call the
+ // function recursively
+ this.LoadDirHBase(file.getAbsolutePath());
+ }
+ }
+
+ private void LoadDirHBase(String dirname) {
+
+ this.dirName = dirname;
+ this.LoadDirHBase();
+
+ }
+
+ /**
+ * @param input
+ * @param hbaseTable
+ * @param col_family
+ * @throws IOException
+ * @throws ParseException
+ *
+ *
+ * Inserts all json records picked up from the inputStream
+ */
+ private void HBaseBulkPut(InputStream input, String col_family,
+ String col_qualifier) throws IOException, ParseException {
+
+ HTable table = new HTable(conf, hbaseTable);
+ JSONParser parser = new JSONParser();
+
+ BufferedReader br = new BufferedReader(new InputStreamReader(input));
+ String jsonString;
+ List<Put> allputs = new ArrayList<Put>();
+ Map json;
+
+ while ((jsonString = br.readLine()) != null) {
+
+ try {
+
+ json = (Map) parser.parse(jsonString);
+ } catch (ParseException e) {
+ // System.out.println("Unable to Parse: " +jsonString);
+ continue;
+ }
+ // Iterator iter = json.entrySet().iterator();
+
+ // Get Address - either IP/domain or email and make that the Key
+ Put put = new Put(Bytes.toBytes((String) json.get("address")));
+
+ // We are just adding a "Y" flag to mark this address
+ put.add(Bytes.toBytes(col_family), Bytes.toBytes(col_qualifier),
+ Bytes.toBytes("Y"));
+
+ allputs.add(put);
+ }
+ table.put(allputs);
+ table.close();
+ }
+
+ private void printUsage() {
+ System.out
+ .println("Usage: java -cp JarFile com.apache.metron.dataloads.cif.HBaseTableLoad -d <directory> -t <tablename> -f <optional file-list>");
+ }
+
+ private void parse(String[] args) {
+ CommandLineParser parser = new BasicParser();
+ Options options = new Options();
+
+ options.addOption("d", true, "description");
+ options.addOption("t", true, "description");
+ options.addOption("f", false, "description");
+
+ CommandLine cmd = null;
+ try {
+ cmd = parser.parse(options, args);
+
+ if (cmd.hasOption("d"))
+ {
+ this.dirName = cmd.getOptionValue("d");
+ LOG.info("Directory Name:" + cmd.getOptionValue("d"));
+ }
+ else {
+ LOG.info("Missing Directory Name");
+ printUsage();
+ System.exit(-1);
+ }
+
+ if (cmd.hasOption("t"))
+ {
+ this.hbaseTable = cmd.getOptionValue("t");
+ LOG.info("HBase Table Name:" + cmd.getOptionValue("t"));
+ }
+ else {
+ LOG.info("Missing Table Name");
+ printUsage();
+ System.exit(-1);
+ }
+
+ if (cmd.hasOption("f")) {
+ this.usefileList = true;
+ files = LoadFileList(cmd.getOptionValue("f"));
+ LOG.info("FileList:" + cmd.getOptionValue("f"));
+ }
+
+ } catch (org.apache.commons.cli.ParseException e) {
+ LOG.error("Failed to parse comand line properties", e);
+ e.printStackTrace();
+ System.exit(-1);
+ }
+ }
+
+ private Set<String> LoadFileList(String filename) {
+
+ Set<String> output = null;
+ BufferedReader reader;
+
+ try {
+ reader = new BufferedReader(new InputStreamReader(
+ new FileInputStream(filename)));
+ output = new HashSet<String>();
+ String in = "";
+
+ while ((in = reader.readLine()) != null)
+ output.add(in);
+
+ reader.close();
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ return output;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsCacheReaper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsCacheReaper.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsCacheReaper.java
deleted file mode 100644
index 64423a6..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsCacheReaper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.apache.metron.alerts.server;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import com.google.inject.Inject;
-
-public class AlertsCacheReaper implements Runnable {
-
- private AlertsSearcher searcher;
-
- @Inject
- private Properties configProps;
-
- public void setSearcher(AlertsSearcher searcher) {
- this.searcher = searcher;
- }
-
- @Override
- public void run() {
-
- long expireAlertsCacheInterval = Long.parseLong( configProps.getProperty( "alerts.cache.expiration.interval", "360000" ) );
- long timeNow = System.currentTimeMillis();
-
- long cutOffTime = timeNow - expireAlertsCacheInterval;
-
- List<String> forRemoval = new ArrayList<String>();
-
- for( Map.Entry<String, AlertsFilterCacheEntry> entry : searcher.alertsFilterCache.entrySet() )
- {
- // if entry was saved more than X timeunits ago, remove it
- if( entry.getValue().storedAtTime < cutOffTime )
- {
- forRemoval.add(entry.getKey());
- }
- }
-
- for( String key : forRemoval )
- {
- searcher.alertsFilterCache.remove(key);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsFilterCacheEntry.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsFilterCacheEntry.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsFilterCacheEntry.java
deleted file mode 100644
index 68fbd28..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsFilterCacheEntry.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.apache.metron.alerts.server;
-
-public class AlertsFilterCacheEntry {
-
-
- public String sourceData;
- public long storedAtTime;
-
-
- public AlertsFilterCacheEntry(String sourceData, long timeNow) {
- this.sourceData = sourceData;
- this.storedAtTime = timeNow;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsProcessingServer.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsProcessingServer.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsProcessingServer.java
deleted file mode 100644
index 1b4a121..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsProcessingServer.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package com.apache.metron.alerts.server;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-
-import java.util.Properties;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-
-public class AlertsProcessingServer {
-
- private static final Logger logger = LoggerFactory.getLogger( AlertsProcessingServer.class );
-
- private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-
- @Inject
- private AlertsSearcher searcher;
- @Inject
- private AlertsCacheReaper reaper;
- @Inject
- private Properties configProps;
-
- public void startProcessing() {
-
- logger.debug( "startProcessing() invoked" );
-
- int initialDelayTime = Integer.parseInt( configProps.getProperty( "searchInitialDelayTime", "30" ) );
- int searchIntervalTime = Integer.parseInt( configProps.getProperty( "searchIntervalTime", "30" ) );
-
- reaper.setSearcher(searcher);
-
- final ScheduledFuture<?> alertsSearcherHandle =
- scheduler.scheduleAtFixedRate( searcher, initialDelayTime, searchIntervalTime, SECONDS );
-
- scheduler.scheduleAtFixedRate(reaper, 120, 380, SECONDS);
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsSearcher.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsSearcher.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsSearcher.java
deleted file mode 100644
index 0fe4ad2..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/alerts/server/AlertsSearcher.java
+++ /dev/null
@@ -1,235 +0,0 @@
-package com.apache.metron.alerts.server;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-
-public class AlertsSearcher implements Runnable {
-
- private static final Logger logger = LoggerFactory.getLogger( AlertsSearcher.class );
-
- @Inject
- private Properties configProps;
-
- // TODO: inject Searcher module for either ElasticSearch or Solr...
- // TODO: inject MetronServiceFactory here
-
- public final Map<String, AlertsFilterCacheEntry> alertsFilterCache = new HashMap<String, AlertsFilterCacheEntry>();
- MessageDigest md;
-
- public AlertsSearcher() throws NoSuchAlgorithmException
- {
- md = MessageDigest.getInstance("SHA-256");
-
- }
-
- @Override
- public void run() {
-
- try
- {
- logger.debug( "Doing Elastic Search search..." );
-
- long currentSearchTime = System.currentTimeMillis();
- long lastSearchTime = 0L;
-
- // look for a marker that tells us the last time we ran...
- String homeDir = configProps.getProperty("homeDir");
- if( homeDir.endsWith( "/" )) {
- homeDir = homeDir.substring(0, homeDir.length()-1);
- }
-
- logger.info( "using homeDir = " + homeDir );
-
- File searcherStateFile = new File( homeDir + "/searcherState.properties" );
- if( searcherStateFile.exists() )
- {
- logger.info( "found existing searcherState.properties file" );
- FileInputStream fis = null;
- try {
- fis = new FileInputStream( searcherStateFile );
- Properties searcherState = new Properties();
- searcherState.load(fis);
- lastSearchTime = Long.parseLong( searcherState.getProperty("lastSearchTime"));
- }
- catch( FileNotFoundException e ) {
- logger.error( "Error locating lastSearchTime value from state file", e );
-
- } catch (IOException e) {
- logger.error( "Error locating lastSearchTime value from state file", e );
- }
- finally
- {
- try {
- fis.close();
- } catch (IOException e) {
- logger.error( "Probably ignorable error closing file stream: ", e );
- }
- }
- }
- else
- {
- // nothing to do here. We'll write out our lastSearchTime at the end
- logger.info( "No existing searcherState.properties found" );
- }
-
- // search for alerts newer than "lastSearchTime"
- Settings settings = ImmutableSettings.settingsBuilder()
- .put("client.transport.sniff", true).build();
- // .put("cluster.name", "elasticsearch").build();
-
- Client client = null;
- try
- {
- logger.info( "initializing elasticsearch client" );
-
- String elasticSearchHostName = configProps.getProperty( "elasticSearchHostName", "localhost" );
- int elasticSearchHostPort = Integer.parseInt(configProps.getProperty( "elasticSearchHostPort", "9300" ) );
- client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress(elasticSearchHostName, elasticSearchHostPort));
-
- logger.info( "lastSearchTime: " + lastSearchTime );
-
- String alertsIndexes = configProps.getProperty( "alertsIndexes", "alerts" );
- String[] alertsIndexArray = alertsIndexes.split(",");
-
- String alertsTypes = configProps.getProperty( "alertsTypes", "" );
- String[] alertsTypesArray = alertsTypes.split( ",");
-
- String alertsQueryFieldName = configProps.getProperty( "alertQueryFieldName", "alert.source" );
- String alertsQueryFieldValue = configProps.getProperty( "alertsQueryFieldValue", "*" );
-
- logger.info( "alertsIndexes: " + alertsIndexes );
-
- String[] foo = new String[1];
-
- SearchResponse response = client.prepareSearch( alertsIndexArray )
- .setTypes( alertsTypesArray )
- .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
- .addField("_source")
- .setQuery( QueryBuilders.boolQuery().must( QueryBuilders.wildcardQuery( alertsQueryFieldName, alertsQueryFieldValue ) )
- .must( QueryBuilders.rangeQuery("message.timestamp").from(lastSearchTime).to(System.currentTimeMillis()).includeLower(true).includeUpper(false)))
- .execute()
- .actionGet();
-
- SearchHits hits = response.getHits();
- logger.debug( "Total hits: " + hits.getTotalHits());
-
-
- // for all hits, put the alert onto the Kafka topic.
- for( SearchHit hit : hits )
- {
- // calculate hash for this hit...
- String sourceData = hit.getSourceAsString();
- String hash = new String( md.digest(sourceData.getBytes()));
-
- if( alertsFilterCache.containsKey(hash))
- {
- logger.warn( "We have already seen this Alert, so skipping..." );
- continue;
- }
- else
- {
- long timeNow = System.currentTimeMillis();
- AlertsFilterCacheEntry cacheEntry = new AlertsFilterCacheEntry( sourceData, timeNow );
- alertsFilterCache.put(hash, cacheEntry);
- }
-
- doSenderWork(hit);
- }
-
- }
- finally
- {
- if( client != null )
- {
- client.close();
- }
- }
-
- // record the time we just searched
- FileOutputStream fos = null;
- try {
- fos = new FileOutputStream( searcherStateFile );
- Properties searcherState = new Properties();
- searcherState.setProperty( "lastSearchTime", Long.toString(currentSearchTime));
- searcherState.store(fos, "");
-
- }
- catch( FileNotFoundException e ) {
- logger.error( "Error saving lastSearchTime: ", e );
- } catch (IOException e) {
- logger.error( "Error saving lastSearchTime: ", e );
- }
- finally {
-
- try {
- fos.close();
- }
- catch (IOException e) {
- logger.error( "Probably ignorable error closing file stream: ", e );
- }
- }
-
- logger.info( "Done with ElasticSearch search... " );
- }
- catch( Exception e )
- {
- logger.error( "Unexpected error while searching ElasticSearch index:", e );
- }
- }
-
- private void doSenderWork( SearchHit hit )
- {
- String kafkaBrokerHostName = configProps.getProperty("kafkaBrokerHostName", "localhost" );
- String kafkaBrokerHostPort = configProps.getProperty("kafkaBrokerHostPort", "9092" );
- String kafkaTopicName = configProps.getProperty("kafkaTopicName", "test" );
-
- logger.debug( "kafkaBrokerHostName: " + kafkaBrokerHostName );
- logger.debug( "kafkaBrokerHostPort: " + kafkaBrokerHostPort );
- logger.debug( "kafkaTopicName: " + kafkaTopicName );
-
- String sourceData = hit.getSourceAsString();
-
- logger.debug( "Source Data: " + sourceData );
- Properties props = new Properties();
-
- props.put("metadata.broker.list", kafkaBrokerHostName + ":" + kafkaBrokerHostPort );
- props.put("serializer.class", "kafka.serializer.StringEncoder");
- // props.put("partitioner.class", "example.producer.SimplePartitioner");
- props.put("request.required.acks", "1");
-
- ProducerConfig config = new ProducerConfig(props);
-
- Producer<String, String> producer = new Producer<String, String>(config);
-
- KeyedMessage<String, String> data = new KeyedMessage<String, String>(kafkaTopicName, "", sourceData );
-
- producer.send(data);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/Main.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/Main.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/Main.java
deleted file mode 100644
index 11d88fa..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/Main.java
+++ /dev/null
@@ -1,288 +0,0 @@
-package com.apache.metron.dataservices;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Properties;
-
-import javax.servlet.DispatcherType;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.jasper.servlet.JspServlet;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.apache.shiro.web.env.EnvironmentLoaderListener;
-import org.apache.tomcat.InstanceManager;
-import org.apache.tomcat.SimpleInstanceManager;
-import org.eclipse.jetty.annotations.ServletContainerInitializersStarter;
-import org.eclipse.jetty.apache.jsp.JettyJasperInitializer;
-import org.eclipse.jetty.plus.annotation.ContainerInitializer;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.SecureRequestCustomizer;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.SslConnectionFactory;
-import org.eclipse.jetty.servlet.DefaultServlet;
-import org.eclipse.jetty.servlet.FilterHolder;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.jboss.resteasy.plugins.guice.GuiceResteasyBootstrapServletContextListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.servlet.GuiceFilter;
-import com.apache.metron.alerts.server.AlertsProcessingServer;
-import com.apache.metron.dataservices.modules.guice.AlertsServerModule;
-import com.apache.metron.dataservices.modules.guice.DefaultServletModule;
-import com.apache.metron.dataservices.modules.guice.DefaultShiroWebModule;
-
-public class Main {
-
- static int port = 9091;
-
- private static final String WEBROOT_INDEX = "/webroot/";
-
- private static final Logger logger = LoggerFactory.getLogger( Main.class );
-
- public static void main(String[] args) throws Exception {
-
-
- Options options = new Options();
-
- options.addOption( "homeDir", true, "Home directory for the service" );
-
- CommandLineParser parser = new GnuParser();
- CommandLine cmd = parser.parse( options, args);
-
- Properties configProps = new Properties();
-
- String homeDir = cmd.getOptionValue("homeDir");
-
- if( homeDir.endsWith( "/" ))
- {
- homeDir = homeDir.substring(0, homeDir.length()-1);
- }
-
-
- DOMConfigurator.configure( homeDir + "/log4j.xml" );
-
- logger.warn( "DataServices Server starting..." );
-
-
- File configFile = new File( homeDir + "/config.properties" );
- FileReader configFileReader = new FileReader( configFile );
- try
- {
- configProps.load(configFileReader);
-
- Option[] cmdOptions = cmd.getOptions();
- for( Option opt : cmdOptions )
- {
- String argName = opt.getOpt();
- String argValue = opt.getValue();
-
- configProps.put(argName, argValue);
- }
-
- }
- finally
- {
- if( configFileReader != null )
- {
- configFileReader.close();
- }
- }
-
- WebAppContext context = new WebAppContext();
-
- Injector injector = Guice.createInjector( new DefaultServletModule(configProps),
- new AlertsServerModule(configProps),
- new DefaultShiroWebModule(configProps, context.getServletContext()),
- new AbstractModule() {
-
- @Override
- protected void configure() {
- binder().requireExplicitBindings();
- bind(GuiceFilter.class);
- bind( GuiceResteasyBootstrapServletContextListener.class );
- bind( EnvironmentLoaderListener.class );
-
- }
- }
- );
-
-
- injector.getAllBindings();
- injector.createChildInjector().getAllBindings();
-
- Server server = new Server(port);
-
- /***************************************************
- *************** enable SSL ************************
- ***************************************************/
-
- // HTTP Configuration
- HttpConfiguration http_config = new HttpConfiguration();
- http_config.setSecureScheme("https");
- http_config.setSecurePort(8443);
- http_config.setOutputBufferSize(32768);
- http_config.setRequestHeaderSize(8192);
- http_config.setResponseHeaderSize(8192);
- http_config.setSendServerVersion(true);
- http_config.setSendDateHeader(false);
- // httpConfig.addCustomizer(new ForwardedRequestCustomizer())
- // SSL Context Factory
- SslContextFactory sslContextFactory = new SslContextFactory();
-
- String sslKeystorePath = configProps.getProperty( "sslKeystorePath", "/keystore" );
- logger.debug( "sslKeystorePath: " + sslKeystorePath );
- sslContextFactory.setKeyStorePath( homeDir + sslKeystorePath );
-
- String sslKeystorePassword = configProps.getProperty( "sslKeystorePassword" );
- sslContextFactory.setKeyStorePassword(sslKeystorePassword);
-
- String sslKeyManagerPassword = configProps.getProperty( "sslKeyManagerPassword" );
- if( sslKeyManagerPassword != null && !sslKeyManagerPassword.isEmpty() )
- {
- sslContextFactory.setKeyManagerPassword(sslKeyManagerPassword);
- }
-
- String sslTruststorePath = configProps.getProperty( "sslTruststorePath" );
- if( sslTruststorePath != null && !sslTruststorePath.isEmpty() )
- {
- sslContextFactory.setTrustStorePath( homeDir + sslTruststorePath );
- }
-
- String sslTruststorePassword = configProps.getProperty( "sslTruststorePassword" );
- if( sslTruststorePassword != null && !sslTruststorePassword.isEmpty())
- {
- sslContextFactory.setTrustStorePassword( sslTruststorePassword );
- }
-
- sslContextFactory.setExcludeCipherSuites(
- "SSL_RSA_WITH_DES_CBC_SHA",
- "SSL_DHE_RSA_WITH_DES_CBC_SHA",
- "SSL_DHE_DSS_WITH_DES_CBC_SHA",
- "SSL_RSA_EXPORT_WITH_RC4_40_MD5",
- "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA",
- "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA",
- "SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA");
-
- // SSL HTTP Configuration
- HttpConfiguration https_config = new HttpConfiguration(http_config);
- https_config.addCustomizer(new SecureRequestCustomizer());
-
- // SSL Connector
- ServerConnector sslConnector = new ServerConnector(server,
- new SslConnectionFactory(sslContextFactory,"http/1.1"),
- new HttpConnectionFactory(https_config));
- sslConnector.setPort(8443);
- server.addConnector(sslConnector);
-
-
- FilterHolder guiceFilter = new FilterHolder(injector.getInstance(GuiceFilter.class));
-
-
- /** For JSP support. Used only for testing and debugging for now. This came come out
- * once the real consumers for this service are in place
- */
- URL indexUri = Main.class.getResource(WEBROOT_INDEX);
- if (indexUri == null)
- {
- throw new FileNotFoundException("Unable to find resource " + WEBROOT_INDEX);
- }
-
- // Points to wherever /webroot/ (the resource) is
- URI baseUri = indexUri.toURI();
-
- // Establish Scratch directory for the servlet context (used by JSP compilation)
- File tempDir = new File(System.getProperty("java.io.tmpdir"));
- File scratchDir = new File(tempDir.toString(),"embedded-jetty-jsp");
-
- if (!scratchDir.exists())
- {
- if (!scratchDir.mkdirs())
- {
- throw new IOException("Unable to create scratch directory: " + scratchDir);
- }
- }
-
- // Set JSP to use Standard JavaC always
- System.setProperty("org.apache.jasper.compiler.disablejsr199","false");
-
- context.setAttribute("javax.servlet.context.tempdir",scratchDir);
- context.setAttribute(InstanceManager.class.getName(), new SimpleInstanceManager());
-
- //Ensure the jsp engine is initialized correctly
- JettyJasperInitializer sci = new JettyJasperInitializer();
- ServletContainerInitializersStarter sciStarter = new ServletContainerInitializersStarter(context);
- ContainerInitializer initializer = new ContainerInitializer(sci, null);
- List<ContainerInitializer> initializers = new ArrayList<ContainerInitializer>();
- initializers.add(initializer);
-
- context.setAttribute("org.eclipse.jetty.containerInitializers", initializers);
- context.addBean(sciStarter, true);
-
- // Set Classloader of Context to be sane (needed for JSTL)
- // JSP requires a non-System classloader, this simply wraps the
- // embedded System classloader in a way that makes it suitable
- // for JSP to use
- // new URL( "file:///home/prhodes/.m2/repository/javax/servlet/jsp/javax.servlet.jsp-api/2.3.1/javax.servlet.jsp-api-2.3.1.jar" )
- ClassLoader jspClassLoader = new URLClassLoader(new URL[] {}, Thread.currentThread().getContextClassLoader());
- context.setClassLoader(jspClassLoader);
-
- // Add JSP Servlet (must be named "jsp")
- ServletHolder holderJsp = new ServletHolder("jsp",JspServlet.class);
- holderJsp.setInitOrder(0);
- holderJsp.setInitParameter("logVerbosityLevel","DEBUG");
- holderJsp.setInitParameter("fork","false");
- holderJsp.setInitParameter("xpoweredBy","false");
- holderJsp.setInitParameter("compilerTargetVM","1.7");
- holderJsp.setInitParameter("compilerSourceVM","1.7");
- holderJsp.setInitParameter("keepgenerated","true");
- context.addServlet(holderJsp,"*.jsp");
- //context.addServlet(holderJsp,"*.jspf");
- //context.addServlet(holderJsp,"*.jspx");
-
- // Add Default Servlet (must be named "default")
- ServletHolder holderDefault = new ServletHolder("default",DefaultServlet.class);
- holderDefault.setInitParameter("resourceBase",baseUri.toASCIIString());
- holderDefault.setInitParameter("dirAllowed","true");
- context.addServlet(holderDefault,"/");
-
- /** end "for JSP support */
-
-
- context.setResourceBase(baseUri.toASCIIString());
-
- context.setInitParameter("resteasy.guice.modules", "com.apache.metron.dataservices.modules.guice.RestEasyModule");
- context.setInitParameter("resteasy.servlet.mapping.prefix", "/rest");
-
- context.addEventListener(injector.getInstance(GuiceResteasyBootstrapServletContextListener.class));
- context.addFilter(guiceFilter, "/*", EnumSet.allOf(DispatcherType.class));
-
- server.setHandler(context);
- server.start();
-
- AlertsProcessingServer alertsServer = injector.getInstance(AlertsProcessingServer.class);
-
- alertsServer.startProcessing();
-
- server.join();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthToken.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthToken.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthToken.java
deleted file mode 100644
index 8add2ec..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthToken.java
+++ /dev/null
@@ -1,187 +0,0 @@
-package com.apache.metron.dataservices.auth;
-
-import java.io.FileInputStream;
-import java.security.KeyStore;
-import java.util.Properties;
-
-import javax.crypto.Cipher;
-import javax.crypto.KeyGenerator;
-import javax.crypto.SecretKey;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.codec.binary.Base64;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class AuthToken {
-
- private static final Logger logger = LoggerFactory.getLogger( AuthToken.class );
-
- public static String generateToken( final Properties configProps ) throws Exception
- {
-
- KeyStore ks = KeyStore.getInstance("JCEKS");
- String keystoreFile = configProps.getProperty( "keystoreFile" );
- logger.info( "keystoreFile: " + keystoreFile );
-
- String keystorePassword = configProps.getProperty( "keystorePassword" );
- logger.info( "keystorePassword: " + keystorePassword );
-
- String keystoreAlias = configProps.getProperty( "authTokenAlias" );
- logger.info( "keystoreAlias: " + keystoreAlias );
-
- FileInputStream fis = null;
- try {
- fis = new FileInputStream( keystoreFile );
- ks.load(fis, keystorePassword.toCharArray() );
- }
- catch( Exception e )
- {
- logger.error( "Error opening keyfile:", e );
- throw e;
- }
- finally {
- fis.close();
- }
-
- KeyStore.ProtectionParameter protParam =
- new KeyStore.PasswordProtection(keystorePassword.toCharArray());
- KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry)ks.getEntry(keystoreAlias, protParam);
-
- SecretKey key = secretKeyEntry.getSecretKey();
-
-
- Cipher cipher = Cipher.getInstance("AES");
- cipher.init(Cipher.ENCRYPT_MODE, key);
- String tokenString = "Metron_AuthToken:" + System.currentTimeMillis();
-
- byte[] encryptedData = cipher.doFinal(tokenString.getBytes());
-
- String base64Token = new String( Base64.encodeBase64(encryptedData) );
-
- // System.out.println( "base64Token: " + base64Token );
-
- return base64Token;
-
- }
-
- public static boolean validateToken( final Properties configProps, String authToken ) throws Exception
- {
- KeyStore ks = KeyStore.getInstance("JCEKS");
- String keystoreFile = configProps.getProperty( "keystoreFile" );
- String keystorePassword = configProps.getProperty( "keystorePassword" );
- String keystoreAlias = configProps.getProperty( "authTokenAlias" );
- long tokenMaxAgeInMilliseconds = Long.parseLong( configProps.getProperty( "authTokenMaxAge", "600000" ));
-
- FileInputStream fis = null;
- try {
- fis = new FileInputStream( keystoreFile );
- ks.load(fis, keystorePassword.toCharArray() );
- }
- finally {
- if( fis != null) {
- fis.close();
- }
- }
-
- KeyStore.ProtectionParameter protParam =
- new KeyStore.PasswordProtection(keystorePassword.toCharArray());
- KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry)ks.getEntry(keystoreAlias, protParam);
-
- SecretKey key = secretKeyEntry.getSecretKey();
-
- Cipher cipher = Cipher.getInstance("AES");
- cipher.init(Cipher.DECRYPT_MODE, key);
-
- byte[] encryptedBytes = Base64.decodeBase64(authToken);
-
- byte[] unencryptedBytes = cipher.doFinal(encryptedBytes);
- String clearTextToken = new String( unencryptedBytes );
-
- System.out.println( "clearTextToken: " + clearTextToken );
- String[] tokenParts = clearTextToken.split( ":" );
-
- if( tokenParts[0].equals( "Metron_AuthToken" ))
- {
- long now = System.currentTimeMillis();
- long tokenTime = Long.parseLong(tokenParts[1]);
-
- if( now > (tokenTime + tokenMaxAgeInMilliseconds ))
- {
- return false;
- }
- else
- {
- return true;
- }
- }
- else
- {
- return false;
- }
-
- }
-
- public static void main( String[] args ) throws Exception
- {
-
- Options options = new Options();
-
- options.addOption( "keystoreFile", true, "Keystore File" );
- options.addOption( "keystorePassword", true, "Keystore Password" );
- options.addOption( "authTokenAlias", true, "");
-
- CommandLineParser parser = new GnuParser();
- CommandLine cmd = parser.parse( options, args);
-
-
- try
- {
- KeyStore ks = KeyStore.getInstance("JCEKS");
-
- String keystorePassword = cmd.getOptionValue("keystorePassword");
- String keystoreFile = cmd.getOptionValue("keystoreFile");
- String authTokenAlias = cmd.getOptionValue("authTokenAlias");
-
- ks.load(null, keystorePassword.toCharArray());
-
-
- // generate a key and store it in the keystore...
- KeyGenerator keyGen = KeyGenerator.getInstance("AES");
- SecretKey key = keyGen.generateKey();
-
- KeyStore.ProtectionParameter protParam =
- new KeyStore.PasswordProtection(keystorePassword.toCharArray());
-
-
- KeyStore.SecretKeyEntry skEntry =
- new KeyStore.SecretKeyEntry(key);
-
- ks.setEntry(authTokenAlias, skEntry, protParam);
-
- java.io.FileOutputStream fos = null;
- try {
-
- fos = new java.io.FileOutputStream(keystoreFile);
- ks.store(fos, keystorePassword.toCharArray());
- }
- finally {
-
- if (fos != null) {
- fos.close();
- }
- }
-
-
- System.out.println( "done" );
-
- }
- catch( Exception e )
- {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3854e075/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthTokenFilter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthTokenFilter.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthTokenFilter.java
deleted file mode 100644
index aab3449..0000000
--- a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/dataservices/auth/AuthTokenFilter.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package com.apache.metron.dataservices.auth;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import javax.ws.rs.NameBinding;
-
-@Target({ ElementType.TYPE, ElementType.METHOD })
-@Retention(value = RetentionPolicy.RUNTIME)
-@NameBinding
-public @interface AuthTokenFilter {
-
-}