You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/04/29 18:28:34 UTC
[4/6] incubator-metron git commit: METRON-119 - Move PCAP
infrastructure from HBase closes apache/incubator-metron#93
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/HBaseIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/HBaseIntegrationTest.java b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/HBaseIntegrationTest.java
deleted file mode 100644
index 62aa721..0000000
--- a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/HBaseIntegrationTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.pcapservice;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * The Class HBaseIntegrationTest.
- *
- * @author Sayi
- */
-public class HBaseIntegrationTest {
-
- /** The test util. */
- private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
-
- /** The test table. */
- private HTable testTable;
-
- /**
- * Inits the cluster.
- *
- * @throws Exception
- * the exception
- */
- void initCluster() throws Exception {
- // testUtil.getConfiguration().addResource("hbase-site-local.xml");
- // testUtil.getConfiguration().reloadConfiguration();
- // start mini hbase cluster
- testUtil.startMiniCluster(1);
- // create tables
- createTable();
-
- }
-
- /**
- * Creates the table.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- private void createTable() throws IOException {
- testTable = testUtil.createTable(Bytes.toBytes("test_pcaps_local"), Bytes.toBytes("cf"));
- System.out.println("after 'test_pcaps_local' table creation ");
- // create put
- Put put = new Put(Bytes.toBytes("1111")); // row key =1111
- put.add(Bytes.toBytes("cf"), Bytes.toBytes("packet"),
- Bytes.toBytes("aaaaaaaa"));
- testTable.put(put);
- System.out.println("after testTable.put(put)");
-
- }
-
- /**
- * The main method.
- *
- * @param args
- * the arguments
- * @throws Exception
- * the exception
- */
- public static void main(String[] args) throws Exception {
- // HBaseIntegrationTest test = new HBaseIntegrationTest();
- // test.initCluster();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapGetterHBaseImplTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapGetterHBaseImplTest.java b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapGetterHBaseImplTest.java
deleted file mode 100644
index 82cb278..0000000
--- a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapGetterHBaseImplTest.java
+++ /dev/null
@@ -1,553 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.pcapservice;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Scan;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.springframework.util.Assert;
-
-import org.apache.metron.pcapservice.PcapGetterHBaseImpl;
-import org.apache.metron.pcapservice.PcapsResponse;
-
-/**
- * The Class PcapGetterHBaseImplTest.
- */
-public class PcapGetterHBaseImplTest {
-
- /**
- * Sets the up.
- *
- * @throws Exception
- * the exception
- */
- @Before
- public void setUp() throws Exception {
- }
-
- /**
- * Tear down.
- *
- * @throws Exception
- * the exception
- */
- @After
- public void tearDown() throws Exception {
- }
-
- /**
- * Test_get pcaps_with list.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @SuppressWarnings("unchecked")
- @Test
- public void test_getPcaps_withList() throws IOException {
- // mocking
- String[] keys = { "0a07002b-0a078039-06-1e8b-0087",
- "0a070025-0a07807a-06-aab8-c360" };
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- PcapGetterHBaseImpl spy = Mockito.spy(pcapGetter);
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
-
- // Mockito.doReturn(mockPcaps).when(spy).scanPcaps(Mockito.any(ArrayList.class),
- // Mockito.any(HTable.class), Mockito.any(Scan.class),
- // Mockito.any(byte[].class), Mockito.any(byte[].class));
- //
- //
- // actual call
- // PcapsResponse response = spy.getPcaps(Arrays.asList(keys));
-
- // verify
- // Assert.assertTrue(response.getResponseSize() == mockPcaps.get(0).length);
- }
-
- /**
- * Test_get pcaps_with key.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @SuppressWarnings("unchecked")
- @Test
- public void test_getPcaps_withKey() throws IOException {
- // mocking
- String key = "0a07002b-0a078039-06-1e8b-0087";
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- PcapGetterHBaseImpl spy = Mockito.spy(pcapGetter);
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
-
- // //
- // Mockito.doReturn(mockPcaps).when(spy).scanPcaps(Mockito.any(ArrayList.class),
- // Mockito.any(HTable.class), Mockito.any(Scan.class),
- // Mockito.any(byte[].class), Mockito.any(byte[].class));
- //
-
- // actual call
- // PcapsResponse response = spy.getPcaps(key);
-
- // verify
- // Assert.assertTrue(response.getResponseSize() == mockPcaps.get(0).length);
- }
-
- /**
- * Test_get pcaps_with key and timestamps.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @SuppressWarnings("unchecked")
- @Test
- public void test_getPcaps_withKeyAndTimestamps() throws IOException {
- // mocking
- String key = "0a07002b-0a078039-06-1e8b-0087";
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- PcapGetterHBaseImpl spy = Mockito.spy(pcapGetter);
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
-
- // Mockito.doReturn(mockPcaps).when(spy).scanPcaps(Mockito.any(ArrayList.class),
- // Mockito.any(HTable.class), Mockito.any(Scan.class),
- // Mockito.any(byte[].class), Mockito.any(byte[].class));
-
- // actual call
- // PcapsResponse response = spy.getPcaps(key, startTime, endTime, false);
-
- // verify
- // Assert.assertTrue(response.getResponseSize() == mockPcaps.get(0).length);
- }
-
- /**
- * Test_get pcaps_with key_multiple pcaps.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @SuppressWarnings("unchecked")
- @Test
- public void test_getPcaps_withKey_multiplePcaps() throws IOException {
- // mocking
- String key = "0a07002b-0a078039-06-1e8b-0087";
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- PcapGetterHBaseImpl spy = Mockito.spy(pcapGetter);
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
- mockPcaps.add(getTestPcapBytes());
-
- /*
- * Mockito.doReturn(mockPcaps).when(spy).scanPcaps(Mockito.any(ArrayList.class
- * ), Mockito.any(HTable.class), Mockito.any(Scan.class),
- * Mockito.any(byte[].class), Mockito.any(byte[].class));
- */
- // actual call
- // PcapsResponse response = spy.getPcaps(key);
-
- // verify
- // Assert.assertNotNull(response);
- // Assert.assertTrue(response.getResponseSize() > mockPcaps.get(0).length);
- }
-
- /**
- * Gets the test pcap bytes.
- *
- * @return the test pcap bytes
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- private byte[] getTestPcapBytes() throws IOException {
- File fin = new File("src/test/resources/test-tcp-packet.pcap");
- byte[] pcapBytes = FileUtils.readFileToByteArray(fin);
- return pcapBytes;
- }
-
- /**
- * Test_remove duplicates.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_removeDuplicates() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
-
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0019-caac");
- keys.add("18800006-1800000b-06-0050-5af6");
-
- List<String> deDupKeys = pcapGetter.removeDuplicateKeys(keys);
- Assert.isTrue(deDupKeys.size() == 3);
- List<String> testKeys = new ArrayList<String>();
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0019-caac");
-
- ListUtils.isEqualList(deDupKeys, testKeys);
- }
-
- /**
- * Test_sort keys by asc order_with out reverse traffic.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_sortKeysByAscOrder_withOutReverseTraffic()
- throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-06-0019-caac");
-
- List<String> result = pcapGetter.sortKeysByAscOrder(keys, false);
-
- List<String> testKeys = new ArrayList<String>();
- testKeys.add("18800006-1800000b-06-0019-caac");
- testKeys.add("18800006-1800000b-06-0050-5af6");
- testKeys.add("18800006-1800000b-11-0035-3810");
-
- Assert.isTrue(ListUtils.isEqualList(result, testKeys));
- }
-
- /**
- * Test_sort keys by asc order_with reverse traffic.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_sortKeysByAscOrder_withReverseTraffic() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
- keys.add("18800006-1800000b-11-0035-3812");
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-11-0035-3811");
-
- List<String> result = pcapGetter.sortKeysByAscOrder(keys, true);
- Assert.isTrue(result.size() == 6);
- }
-
- /**
- * Test_sort keys by asc order_get unprocessed sublist of keys.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_sortKeysByAscOrder_getUnprocessedSublistOfKeys()
- throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-06-0019-caac");
- System.out.println("original keys =" + keys.toString());
-
- List<String> sortedKeys = pcapGetter.sortKeysByAscOrder(keys, false);
- System.out.println("after sortKeysByAscOrder =" + sortedKeys.toString());
-
- List<String> unprocessedKeys1 = pcapGetter.getUnprocessedSublistOfKeys(
- sortedKeys, "18800006-1800000b-06-0019-caac-65140-40815");
- System.out.println("unprocessedKeys1 =" + unprocessedKeys1);
- Assert.isTrue(unprocessedKeys1.size() == 2);
-
- List<String> unprocessedKeys2 = pcapGetter.getUnprocessedSublistOfKeys(
- sortedKeys, "18800006-1800000b-06-0050-5af6-65140-40815");
- // System.out.println("unprocessedKeys2 ="+unprocessedKeys2);
- Assert.isTrue(unprocessedKeys2.size() == 1);
-
- List<String> unprocessedKeys3 = pcapGetter.getUnprocessedSublistOfKeys(
- sortedKeys, "18800006-1800000b-11-0035-3810-6514040815");
- // System.out.println("unprocessedKeys3 ="+unprocessedKeys3);
- Assert.isTrue(unprocessedKeys3.size() == 0);
-
- }
-
- /**
- * Test_sort keys by asc order_get unprocessed sublist of keys_with out match.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_sortKeysByAscOrder_getUnprocessedSublistOfKeys_withOutMatch()
- throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-06-0019-caac");
- System.out.println("original keys =" + keys.toString());
-
- List<String> sortedKeys = pcapGetter.sortKeysByAscOrder(keys, false);
- System.out.println("after sortKeysByAscOrder =" + sortedKeys.toString());
-
- List<String> unprocessedKeys1 = pcapGetter.getUnprocessedSublistOfKeys(
- sortedKeys, "18800006-1800000b-11-89-455-65140-40815");
- System.out.println("unprocessedKeys1 =" + unprocessedKeys1);
- Assert.isTrue(unprocessedKeys1.size() == 3);
- }
-
- /**
- * Test_create start and stop row keys.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createStartAndStopRowKeys() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String key = "18800006-1800000b-11-0035-3810";
- Map<String, String> map = pcapGetter.createStartAndStopRowKeys(key, false,
- false);
- System.out.println("map =" + map.toString());
-
- String lastRowKey = "18800006-1800000b-11-0035-3810-23234-32423";
- Map<String, String> map1 = pcapGetter.createStartAndStopRowKeys(
- lastRowKey, true, false);
- System.out.println("map1 =" + map1.toString());
-
- String lastRowKey2 = "18800006-1800000b-11-0035-3810-23234-32423";
- Map<String, String> map2 = pcapGetter.createStartAndStopRowKeys(
- lastRowKey2, true, true);
- System.out.println("map2 =" + map2.toString());
-
- }
-
- /**
- * Test_check if valid input_valid.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_checkIfValidInput_valid() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- List<String> keys = new ArrayList<String>();
- keys.add("18800006-1800000b-11-0035-3810");
- keys.add("18800006-1800000b-06-0050-5af6");
- keys.add("18800006-1800000b-06-0019-caac");
-
- String lastRowKey = "18800006-1800000b-11-0035-3810-23234-32423";
-
- boolean response = pcapGetter.checkIfValidInput(keys, lastRowKey);
- Assert.isTrue(response);
-
- }
-
- /**
- * Test_check if valid input_in valid.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_checkIfValidInput_inValid() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- @SuppressWarnings("unchecked")
- boolean response = pcapGetter.checkIfValidInput(Collections.EMPTY_LIST,
- null);
- Assert.isTrue(!response);
-
- }
-
- /**
- * Test_check if valid input_valid_mixed.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_checkIfValidInput_valid_mixed() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String lastRowKey = "18800006-1800000b-11-0035-3810-23234-32423";
- @SuppressWarnings("unchecked")
- boolean response = pcapGetter.checkIfValidInput(Collections.EMPTY_LIST,
- lastRowKey);
- Assert.isTrue(response);
- }
-
- /**
- * Test_create get request.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createGetRequest() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String key = "18800006-1800000b-11-0035-3810-23234-324230";
-
- long startTime = 139812323L; // in seconds
- long endTime = 139923424L; // in seconds
-
- Get get = pcapGetter.createGetRequest(key, startTime, endTime);
- Assert.notNull(get);
-
- Assert.isTrue(Arrays.equals(get.getRow(), key.getBytes()));
- // compare in micros as the data creation time unit is set to Micros in
- // properties file.
- Assert.isTrue(get.getTimeRange().getMin() == startTime * 1000 );
- Assert.isTrue(get.getTimeRange().getMax() == endTime * 1000 );
- }
-
- /**
- * Test_create get request_default time range.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createGetRequest_defaultTimeRange() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String key = "18800006-1800000b-11-0035-3810-23234-324230";
-
- Get get = pcapGetter.createGetRequest(key, -1, -1);
- Assert.notNull(get);
-
- Assert.isTrue(Arrays.equals(get.getRow(), key.getBytes()));
- Assert.isTrue(get.getTimeRange().getMin() == 0);
- }
-
- /**
- * Test_create get request_with start time.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createGetRequest_withStartTime() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String key = "18800006-1800000b-11-0035-3810-23234-324230";
-
- long startTime = 139812323L; // in seconds
-
- Get get = pcapGetter.createGetRequest(key, startTime, -1);
- Assert.notNull(get);
-
- Assert.isTrue(Arrays.equals(get.getRow(), key.getBytes()));
- Assert.isTrue(get.getTimeRange().getMin() == startTime * 1000 );
- Assert.isTrue(get.getTimeRange().getMax() == Long.valueOf(Long.MAX_VALUE));
- }
-
- /**
- * Test_create get request_with end time.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createGetRequest_withEndTime() throws IOException {
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
- String key = "18800006-1800000b-11-0035-3810-23234-324230";
-
- long endTime = 139923424L; // in seconds
-
- Get get = pcapGetter.createGetRequest(key, -1, endTime);
- Assert.notNull(get);
-
- Assert.isTrue(Arrays.equals(get.getRow(), key.getBytes()));
- Assert.isTrue(get.getTimeRange().getMin() == 0);
- Assert.isTrue(get.getTimeRange().getMax() == endTime * 1000 );
- }
-
- /**
- * Test_create scan request.
- *
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- @Test
- public void test_createScanRequest() throws IOException {
- // mocking
- PcapGetterHBaseImpl pcapGetter = (PcapGetterHBaseImpl) PcapGetterHBaseImpl
- .getInstance();
-
- PcapsResponse pcapsResponse = new PcapsResponse();
-
- Map<String, String> keysMap = new HashMap<String, String>();
- String startKey = "0a07002b-0a078039-06-1e8b-0087-00000-00000";
- String endKey = "0a070025-0a07807a-06-aab8-c360-99999-99999";
- keysMap.put("startKey", startKey);
- keysMap.put("endKey", endKey);
-
- long startTime = 139812323L; // in seconds
- long endTime = 139923424L; // in seconds
- long maxResultSize = 673424;
-
- // actual call
- Scan scan = pcapGetter.createScanRequest(pcapsResponse, keysMap, startTime,
- endTime, maxResultSize);
-
- // verify time range
- Assert.isTrue(scan.getTimeRange().getMin() == startTime * 1000 ); // compare
- // in
- // millis
- Assert.isTrue(scan.getTimeRange().getMax() == endTime * 1000 ); // compare
- // in
- // millis
-
- // verify start and stop rows
- Assert.isTrue(Arrays.equals(scan.getStartRow(), startKey.getBytes()));
- Assert.isTrue(Arrays.equals(scan.getStopRow(), endKey.getBytes()));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapHelperTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapHelperTest.java b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapHelperTest.java
deleted file mode 100644
index 73db384..0000000
--- a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapHelperTest.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.pcapservice;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import org.apache.metron.pcapservice.PcapHelper;
-import org.apache.metron.pcapservice.PcapHelper.TimeUnit;
-import org.springframework.util.Assert;
-
-// TODO: Auto-generated Javadoc
-/**
- * The Class PcapHelperTest.
- *
- * @author Sayi
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(PcapHelper.class)
-public class PcapHelperTest {
-
- /**
- * Sets the up.
- *
- * @throws Exception
- * the exception
- */
- @Before
- public void setUp() throws Exception {
- PowerMockito.spy(PcapHelper.class);
- }
-
- /**
- * Tear down.
- *
- * @throws Exception
- * the exception
- */
- @After
- public void tearDown() throws Exception {
- }
-
- /**
- * Input time is in SECONDS and data creation time is in SECONDS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_seconds() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.SECONDS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222L; // input time in seconds
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222L == time);
- }
-
- /**
- * Input time is in MILLIS and data creation time is in SECONDS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_millis_seconds() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.SECONDS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222333L; // input time in millis
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222L == time);
- }
-
- /**
- * Input time is in MICROS and data creation time is in SECONDS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_micros_seconds() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.SECONDS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222333444L; // input time in micros
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MILLIS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_millis() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MILLIS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222L; // input time in seconds
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222000L == time);
- }
-
- /**
- * Input time is in MILLIS and data creation time is in MILLIS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_millis_millis() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MILLIS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 111112222233L; // input time in millis
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(111112222233L == time);
- }
-
- /**
- * Input time is in MICROS and data creation time is in MILLIS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_micros_millis() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MILLIS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 111112222233344L; // input time in micros
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(111112222233L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_micros() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222L; // input time in seconds
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222000000L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_micros_random() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 13388; // input time in seconds
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(13388000000L == time);
- }
-
- /**
- * Input time is in MILLIS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_millis_micros() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 111112222233L; // input time in millis
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(111112222233000L == time);
- }
-
- /**
- * Input time is in MICROS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_micros_micros() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1111122222334444L; // input time in micros
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1111122222334444L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_micros_0() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 0; // input time in micros
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(0 == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_micros_1() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = 1; // input time in micros
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(1000000L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_seconds_micros_decimal() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long inputTime = 13; // input time in seconds (double to long type casting)
- long time = PcapHelper.convertSecondsToDataCreationTimeUnit(inputTime);
-
- Assert.isTrue(13000000L == time);
- }
-
- /**
- * Input time is in SECONDS and data creation time is in MICROS.
- */
- @Test
- public void test_convertToDataCreationTimeUnit_() {
- PowerMockito.when(PcapHelper.getDataCreationTimeUnit()).thenReturn(
- TimeUnit.MICROS);
- PowerMockito.verifyNoMoreInteractions();
-
- long endTime = (long) 111.333; // input time in seconds (double to long type
- // casting)
- long time = PcapHelper.convertToDataCreationTimeUnit(endTime);
-
- Assert.isTrue(111000000L == time);
- }
-
- /**
- * Test_get data creation time unit.
- */
- @Test
- public void test_getDataCreationTimeUnit() {
- TimeUnit dataCreationTimeUnit = PcapHelper.getDataCreationTimeUnit();
- Assert.isTrue(TimeUnit.MILLIS == dataCreationTimeUnit);
- }
-
- /**
- * Test_reverse key_valid.
- */
- @Test
- public void test_reverseKey_valid() {
- String key = "162.242.152.24-162.242.153.12-TCP-38190-9092";
- String reversekey = PcapHelper.reverseKey(key);
- Assert.isTrue("162.242.153.12-162.242.152.24-TCP-9092-38190"
- .equals(reversekey));
- }
-
- /**
- * Test_reverse key_valid_with fragment.
- */
- @Test
- public void test_reverseKey_valid_withFragment() {
- String key = "162.242.152.24-162.242.153.12-TCP-38190-9092-fragmentId";
- String reversekey = PcapHelper.reverseKey(key);
- Assert.isTrue("162.242.153.12-162.242.152.24-TCP-9092-38190"
- .equals(reversekey));
- }
-
- /**
- * Test_reverse key_in valid.
- */
- @Test
- public void test_reverseKey_inValid() {
- String key = "162.242.152.24-162.242.153.12-TCP-38190-9092-ipId-fragmentId-extra";
- String reversekey = PcapHelper.reverseKey(key);
- Assert.isTrue("".equals(reversekey));
- }
-
- /**
- * Test_reverse key_as list.
- */
- @Test
- public void test_reverseKey_asList() {
- String[] keys = {
- "162.242.152.24-162.242.153.12-TCP-38190-9092-fragmentId",
- "162.242.152.24-162.242.153.12-UDP-38190-9092" };
-
- List<String> reverseKeys = PcapHelper.reverseKey(Arrays.asList(keys));
-
- Assert.isTrue("162.242.153.12-162.242.152.24-TCP-9092-38190"
- .equals(reverseKeys.get(0)));
- Assert.isTrue("162.242.153.12-162.242.152.24-UDP-9092-38190"
- .equals(reverseKeys.get(1)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasyTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasyTest.java b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasyTest.java
new file mode 100644
index 0000000..eab4998
--- /dev/null
+++ b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasyTest.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.pcapservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.timestamp.TimestampConverters;
+import org.apache.metron.pcap.mr.PcapJob;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.EnumMap;
+import java.util.List;
+
+public class PcapReceiverImplRestEasyTest {
+ public static class MockQueryHandler extends PcapJob {
+ Path basePath;
+ Path baseOutputPath;
+ long beginNS;
+ long endNS;
+ EnumMap<Constants.Fields, String> fields;
+ @Override
+ public List<byte[]> query( Path basePath
+ , Path baseOutputPath
+ , long beginNS
+ , long endNS
+ , EnumMap<Constants.Fields, String> fields
+ , Configuration conf
+ , FileSystem fs
+ ) throws IOException, ClassNotFoundException, InterruptedException
+ {
+ this.basePath = basePath;
+ this.baseOutputPath = baseOutputPath;
+ this.beginNS = beginNS;
+ this.endNS = endNS;
+ this.fields = fields;
+ return null;
+ }
+ }
+ final MockQueryHandler queryHandler = new MockQueryHandler();
+ PcapReceiverImplRestEasy restEndpoint = new PcapReceiverImplRestEasy() {{
+ this.queryUtil = queryHandler;
+ }};
+
+ @Before
+ public void setup() throws Exception {
+ ConfigurationUtil.setPcapOutputPath("/output");
+ ConfigurationUtil.setTempQueryOutputPath("/tmp");
+ }
+
+ @Test
+ public void testNormalPath() throws Exception {
+ String srcIp = "srcIp";
+ String dstIp = "dstIp";
+ String protocol = "protocol";
+ String srcPort = "80";
+ String dstPort = "100";
+ long startTime = 100;
+ long endTime = 1000;
+
+ {
+ boolean includeReverseTraffic = false;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(startTime), queryHandler.beginNS);
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(endTime), queryHandler.endNS);
+ Assert.assertEquals(includeReverseTraffic, Boolean.getBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+ {
+ boolean includeReverseTraffic = true;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(startTime), queryHandler.beginNS);
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(endTime), queryHandler.endNS);
+ Assert.assertEquals(includeReverseTraffic, Boolean.parseBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+ }
+
+ @Test
+ public void testNullSrcIp() throws Exception {
+ String srcIp = null;
+ String dstIp = "dstIp";
+ String protocol = "protocol";
+ String srcPort = "80";
+ String dstPort = "100";
+ long startTime = 100;
+ long endTime = 1000;
+ boolean includeReverseTraffic = false;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(startTime), queryHandler.beginNS);
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(endTime), queryHandler.endNS);
+ Assert.assertEquals(includeReverseTraffic, Boolean.getBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+
+ @Test
+ public void testNullDstIp() throws Exception {
+ String srcIp = "srcIp";
+ String dstIp = null;
+ String protocol = "protocol";
+ String srcPort = "80";
+ String dstPort = "100";
+ long startTime = 100;
+ long endTime = 1000;
+ boolean includeReverseTraffic = false;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(startTime), queryHandler.beginNS);
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(endTime), queryHandler.endNS);
+ Assert.assertEquals(includeReverseTraffic, Boolean.getBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+
+ @Test
+ public void testEmptyStartTime() throws Exception {
+ String srcIp = "srcIp";
+ String dstIp = null;
+ String protocol = "protocol";
+ String srcPort = "80";
+ String dstPort = "100";
+ long startTime = -1;
+ long endTime = 1000;
+ boolean includeReverseTraffic = false;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(0, queryHandler.beginNS);
+ Assert.assertEquals(TimestampConverters.MILLISECONDS.toNanoseconds(endTime), queryHandler.endNS);
+ Assert.assertEquals(includeReverseTraffic, Boolean.getBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+
+ @Test
+ public void testEmptyEndTime() throws Exception {
+ String srcIp = "srcIp";
+ String dstIp = null;
+ String protocol = "protocol";
+ String srcPort = "80";
+ String dstPort = "100";
+ long startTime = -1;
+ long endTime = -1;
+ boolean includeReverseTraffic = false;
+ restEndpoint.getPcapsByIdentifiers(srcIp, dstIp, protocol, srcPort, dstPort, startTime, endTime, includeReverseTraffic, null);
+ Assert.assertEquals(new Path(ConfigurationUtil.getPcapOutputPath()), queryHandler.basePath);
+ Assert.assertEquals(new Path(ConfigurationUtil.getTempQueryOutputPath()), queryHandler.baseOutputPath);
+ Assert.assertEquals(srcIp, queryHandler.fields.get(Constants.Fields.SRC_ADDR));
+ Assert.assertEquals(dstIp, queryHandler.fields.get(Constants.Fields.DST_ADDR));
+ Assert.assertEquals(srcPort, queryHandler.fields.get(Constants.Fields.SRC_PORT));
+ Assert.assertEquals(dstPort, queryHandler.fields.get(Constants.Fields.DST_PORT));
+ Assert.assertEquals(0, queryHandler.beginNS);
+ Assert.assertTrue(queryHandler.endNS > 0);
+ Assert.assertEquals(includeReverseTraffic, Boolean.getBoolean(queryHandler.fields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapScannerHBaseImplTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapScannerHBaseImplTest.java b/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapScannerHBaseImplTest.java
deleted file mode 100644
index afac1d3..0000000
--- a/metron-platform/metron-api/src/test/java/org/apache/metron/pcapservice/PcapScannerHBaseImplTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.pcapservice;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Scan;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-// TODO: Auto-generated Javadoc
-/**
- * The Class PcapScannerHBaseImplTest.
- */
-public class PcapScannerHBaseImplTest {
-
- /**
- * Sets the up.
- *
- * @throws Exception
- * the exception
- */
- @Before
- public void setUp() throws Exception {
- }
-
- /**
- * Tear down.
- *
- * @throws Exception
- * the exception
- */
- @After
- public void tearDown() throws Exception {
- }
-
- /**
- * Test_create scan request.
- *
- * @throws IOException
- * the IO exception
- */
- @Test
- public void test_createScanRequest() throws IOException {
- // mocking
- PcapScannerHBaseImpl pcapScanner = (PcapScannerHBaseImpl) PcapScannerHBaseImpl
- .getInstance();
- byte[] cf = "cf".getBytes();
- byte[] cq = "pcap".getBytes();
- String startKey = "0a07002b-0a078039-06-1e8b-0087";
- String endKey = "0a070025-0a07807a-06-aab8-c360";
- long maxResultSize = 60;
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
-
- // actual call
- Scan scan = pcapScanner.createScanRequest(cf, cq, startKey, endKey,
- maxResultSize, -1, -1);
-
- // verify
- Assert.assertTrue(scan.getTimeRange().getMin() == 0);
- Assert.assertTrue(Arrays.equals(scan.getStartRow(), startKey.getBytes()));
- Assert.assertTrue(Arrays.equals(scan.getStopRow(), endKey.getBytes()));
- }
-
- /**
- * Test_create scan request_with timestamps.
- *
- * @throws IOException
- * the IO exception
- */
- @Test
- public void test_createScanRequest_withTimestamps() throws IOException {
- // mocking
- PcapScannerHBaseImpl pcapScanner = (PcapScannerHBaseImpl) PcapScannerHBaseImpl
- .getInstance();
- byte[] cf = "cf".getBytes();
- byte[] cq = "pcap".getBytes();
- String startKey = "0a07002b-0a078039-06-1e8b-0087";
- String endKey = "0a070025-0a07807a-06-aab8-c360";
- long maxResultSize = 60;
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
-
- // actual call
- Scan scan = pcapScanner.createScanRequest(cf, cq, startKey, endKey,
- maxResultSize, startTime, endTime);
-
- // verify
- Assert.assertTrue(scan.getTimeRange().getMin() == 1376782349234L);
- Assert.assertTrue(Arrays.equals(scan.getStartRow(), startKey.getBytes()));
- Assert.assertTrue(Arrays.equals(scan.getStopRow(), endKey.getBytes()));
- }
-
- /**
- * Test_get pcaps_with all arguments.
- *
- * @throws IOException
- * the IO exception
- */
- @SuppressWarnings({ "unchecked", "unused" })
- @Test
- public void test_getPcaps_withAllArguments() throws IOException {
- // mocking
- PcapScannerHBaseImpl pcapScanner = (PcapScannerHBaseImpl) PcapScannerHBaseImpl
- .getInstance();
- PcapScannerHBaseImpl spy = Mockito.spy(pcapScanner);
- byte[] cf = "cf".getBytes();
- byte[] cq = "pcap".getBytes();
- String startKey = "0a07002b-0a078039-06-1e8b-0087";
- String endKey = "0a070025-0a07807a-06-aab8-c360";
- long maxResultSize = 60;
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
-
- Mockito
- .doReturn(mockPcaps)
- .when(spy)
- .scanPcaps(Mockito.any(ArrayList.class), Mockito.any(HTable.class),
- Mockito.any(Scan.class), Mockito.any(byte[].class),
- Mockito.any(byte[].class));
-
- // actual call
- byte[] response = spy.getPcaps(startKey, endKey, maxResultSize, startTime,
- endTime);
-
- // verify
- Assert.assertTrue(response.length == mockPcaps.get(0).length);
- }
-
- /**
- * Test_get pcaps_with minimal arguments.
- *
- * @throws IOException
- * the IO exception
- */
- @SuppressWarnings({ "unchecked", "unused" })
- @Test
- public void test_getPcaps_withMinimalArguments() throws IOException {
- // mocking
- PcapScannerHBaseImpl pcapScanner = (PcapScannerHBaseImpl) PcapScannerHBaseImpl
- .getInstance();
- PcapScannerHBaseImpl spy = Mockito.spy(pcapScanner);
- byte[] cf = "cf".getBytes();
- byte[] cq = "pcap".getBytes();
- String startKey = "0a07002b-0a078039-06-1e8b-0087";
- String endKey = "0a070025-0a07807a-06-aab8-c360";
- long maxResultSize = 60;
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
-
- Mockito
- .doReturn(mockPcaps)
- .when(spy)
- .scanPcaps(Mockito.any(ArrayList.class), Mockito.any(HTable.class),
- Mockito.any(Scan.class), Mockito.any(byte[].class),
- Mockito.any(byte[].class));
-
- // actual call
- byte[] response = spy.getPcaps(startKey, endKey);
-
- // verify
- Assert.assertTrue(response.length == mockPcaps.get(0).length);
- }
-
- /**
- * Test_get pcaps_multiple pcaps.
- *
- * @throws IOException
- * the IO exception
- */
- @SuppressWarnings({ "unchecked", "unused" })
- @Test
- public void test_getPcaps_multiplePcaps() throws IOException {
- // mocking
- PcapScannerHBaseImpl pcapScanner = (PcapScannerHBaseImpl) PcapScannerHBaseImpl
- .getInstance();
- PcapScannerHBaseImpl spy = Mockito.spy(pcapScanner);
- byte[] cf = "cf".getBytes();
- byte[] cq = "pcap".getBytes();
- String startKey = "0a07002b-0a078039-06-1e8b-0087";
- String endKey = "0a070025-0a07807a-06-aab8-c360";
- long maxResultSize = 60;
- long startTime = 1376782349234555L;
- long endTime = 1396782349234555L;
-
- List<byte[]> mockPcaps = new ArrayList<byte[]>();
- mockPcaps.add(getTestPcapBytes());
- mockPcaps.add(getTestPcapBytes());
-
- Mockito
- .doReturn(mockPcaps)
- .when(spy)
- .scanPcaps(Mockito.any(ArrayList.class), Mockito.any(HTable.class),
- Mockito.any(Scan.class), Mockito.any(byte[].class),
- Mockito.any(byte[].class));
-
- // actual call
- byte[] response = spy.getPcaps(startKey, endKey);
-
- // verify
- Assert.assertNotNull(response);
- Assert.assertTrue(response.length > mockPcaps.get(0).length);
- }
-
- /**
- * Gets the test pcap bytes.
- *
- * @return the test pcap bytes
- * @throws IOException
- * the IO exception
- */
- private byte[] getTestPcapBytes() throws IOException {
- File fin = new File("src/test/resources/test-tcp-packet.pcap");
- byte[] pcapBytes = FileUtils.readFileToByteArray(fin);
- return pcapBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-api/src/test/resources/hbase-config.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/src/test/resources/hbase-config.properties b/metron-platform/metron-api/src/test/resources/hbase-config.properties
deleted file mode 100644
index 89dbf06..0000000
--- a/metron-platform/metron-api/src/test/resources/hbase-config.properties
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#hbase zoo keeper configuration
-hbase.zookeeper.quorum=dn1.hw.com,dn2.hw.com,dn3.hw.com
-hbase.zookeeper.clientPort=2181
-hbase.client.retries.number=1
-zookeeper.session.timeout=60000
-zookeeper.recovery.retry=0
-
-#hbase table configuration
-hbase.table.name=pcap
-hbase.table.column.family=t
-hbase.table.column.qualifier=pcap
-hbase.table.column.maxVersions=5
-
-# scan size limit configuration in MB or KB; if the input is negative or greater than max value throw an error.
-hbase.scan.result.size.unit=MB
-hbase.scan.default.result.size=6
-hbase.scan.max.result.size=60
-
-# time stamp conversion configuration; possible values 'SECONDS'(seconds), 'MILLIS'(milli seconds), 'MICROS' (micro seconds)
-hbase.table.data.time.unit=MILLIS
-
-#number of retries in case of ZooKeeper or HBase server down
-hbase.hconnection.retries.number=3
-
-#configuration for including pcaps in the reverse traffic
-pcaps.include.reverse.traffic = false
-
-#maximum table row size in KB or MB
-hbase.table.row.size.unit = KB
-hbase.table.max.row.size = 0.07
-
-# tokens of row key configuration
-hbase.table.row.key.tokens=7
-rest.api.input.key.min.tokens=5
-
-# whether or not to include the last row from the previous request, applicable for only partial response scenario
-hbase.table.scan.include.duplicate.lastrow= true;
-
-#number of digits for appending tokens of the row key
-hbase.table.row.key.token.appending.digits=5
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 8050418..91d50fe 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -149,6 +149,7 @@
<artifactId>json-schema-validator</artifactId>
<version>${global_json_schema_validator_version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
@@ -217,7 +218,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
<configuration>
- <targetJdk>1.7</targetJdk>
+ <targetJdk>${global_java_version}</targetJdk>
</configuration>
</plugin>
</plugins>
@@ -229,15 +230,15 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
- <source>1.7</source>
+ <source>${global_java_version}</source>
<compilerArgument>-Xlint:unchecked</compilerArgument>
- <target>1.7</target>
+ <target>${global_java_version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>1.4</version>
+ <version>${global_shade_version}</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<artifactSet>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/assembly/assembly.xml b/metron-platform/metron-common/src/main/assembly/assembly.xml
index 305f0a6..0b36f9d 100644
--- a/metron-platform/metron-common/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-common/src/main/assembly/assembly.xml
@@ -20,7 +20,7 @@
<fileSets>
<fileSet>
<directory>${project.basedir}/src/main/scripts</directory>
- <outputDirectory>/scripts</outputDirectory>
+ <outputDirectory>/bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
<exclude>**/*.formatted</exclude>
@@ -28,6 +28,15 @@
</excludes>
<fileMode>0755</fileMode>
<lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
</fileSet>
</fileSets>
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index 3418e9c..4c7c222 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -29,6 +29,25 @@ public class Constants {
public static final String SENSOR_TYPE = "source.type";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String ERROR_STREAM = "error";
+
+ public static enum Fields {
+ SRC_ADDR("ip_src_addr")
+ ,SRC_PORT("ip_src_port")
+ ,DST_ADDR("ip_dst_addr")
+ ,DST_PORT("ip_dst_port")
+ ,PROTOCOL("protocol")
+ ,TIMESTAMP("timestamp")
+ ,INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic")
+ ;
+ private String name;
+ Fields(String name) {
+ this.name = name;
+ }
+ public String getName() {
+ return name;
+ }
+ }
+
public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment";
public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel";
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java
new file mode 100644
index 0000000..97085b6
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/spout/kafka/SpoutConfig.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.spout.kafka;
+
+import storm.kafka.BrokerHosts;
+
+public class SpoutConfig extends storm.kafka.SpoutConfig {
+ private static enum Offset {
+ BEGINNING, END, WHERE_I_LEFT_OFF;
+ }
+ public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) {
+ super(hosts, topic, zkRoot, id);
+ }
+
+ public SpoutConfig from(String offset) {
+ try {
+ Offset o = Offset.valueOf(offset);
+ from(o);
+ }
+ catch(IllegalArgumentException iae) {
+ from(Offset.WHERE_I_LEFT_OFF);
+ }
+ ignoreZkOffsets = true;
+ startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
+ return this;
+ }
+ private void from(Offset offset) {
+ if(offset == Offset.BEGINNING) {
+ ignoreZkOffsets = true;
+ startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
+ }
+ else if(offset == Offset.END) {
+ ignoreZkOffsets = true;
+ startOffsetTime = kafka.api.OffsetRequest.LatestTime();
+ }
+ else if(offset == Offset.WHERE_I_LEFT_OFF) {
+ ignoreZkOffsets = false;
+ startOffsetTime = kafka.api.OffsetRequest.LatestTime();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java
new file mode 100644
index 0000000..a02e31e
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverter.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils.timestamp;
+
+import java.io.Serializable;
+
+public interface TimestampConverter extends Serializable {
+ public long toNanoseconds(long input);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java
new file mode 100644
index 0000000..1c3aa7f
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/timestamp/TimestampConverters.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils.timestamp;
+
+
+public enum TimestampConverters implements TimestampConverter{
+ MILLISECONDS(tsMilli -> tsMilli*1000000L)
+ ,MICROSECONDS(tsMicro -> tsMicro*1000L)
+ ,NANOSECONDS(tsNano -> tsNano);
+ TimestampConverter converter;
+ TimestampConverters(TimestampConverter converter) {
+ this.converter = converter;
+ }
+
+ public static TimestampConverter getConverter(String converter) {
+ return TimestampConverters.valueOf(converter).converter;
+ }
+
+ @Override
+ public long toNanoseconds(long in) {
+ return converter.toNanoseconds(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
index 4a928bd..bd9e9eb 100755
--- a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
+++ b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh
@@ -26,8 +26,8 @@ if [ -e /usr/libexec/bigtop-detect-javahome ]; then
elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
-export METRON_VERSION=0.1BETA
+export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
-export PARSERS_JAR=metron-parsers-$METRON_VERSION.jar
-export ZK_HOME=${ZK_HOME:-/usr/hdp/current/hbase-client}
-java -cp $METRON_HOME/lib/$PARSERS_JAR org.apache.metron.common.cli.ConfigurationsUtils "$@"
+export PARSERS_JAR=${project.artifactId}-$METRON_VERSION.jar
+export ZK_HOME=${ZK_HOME:-/usr/hdp/current/zookeeper-client}
+java -cp $METRON_HOME/lib/$PARSERS_JAR:$ZK_HOME/lib/* org.apache.metron.common.cli.ConfigurationsUtils "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 6c3f866..941c265 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -118,6 +118,7 @@
<version>${global_hbase_version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
@@ -202,6 +203,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -254,7 +259,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <argLine>-Xmx2048m -XX:MaxPermSize=256m -XX:-UseSplitVerifier</argLine>
+ <argLine>-Xmx2048m</argLine>
<skip>true</skip>
<trimStackTrace>false</trimStackTrace>
</configuration>
@@ -263,14 +268,14 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${global_java_version}</source>
+ <target>${global_java_version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
+ <version>${global_shade_version}</version>
<executions>
<execution>
<phase>package</phase>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
index e464984..03fd641 100755
--- a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
+++ b/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
@@ -36,4 +36,7 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
fi
done
export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
index aed6782..c3f1d05 100644
--- a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
+++ b/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
@@ -17,5 +17,5 @@
# limitations under the License.
#
-yarn jar /usr/metron/${project.version}/lib/metron-data-management-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
index b37e022..b8d9372 100644
--- a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
+++ b/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
@@ -17,5 +17,5 @@
# limitations under the License.
#
-yarn jar /usr/metron/${project.version}/lib/metron-data-management-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
index 2df4ee3..865d0ad 100755
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
+++ b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
@@ -35,4 +35,7 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
fi
done
export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
index c3ec233..6156027 100755
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
+++ b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
@@ -34,4 +34,7 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
fi
done
export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
index 321041a..23d09ba 100755
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
+++ b/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
@@ -36,4 +36,7 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
fi
done
export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/HDFSDataPrunerTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/HDFSDataPrunerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/HDFSDataPrunerTest.java
index 9bc695c..bf5c87e 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/HDFSDataPrunerTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/HDFSDataPrunerTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.dataloads.bulk;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -100,25 +101,29 @@ public class HDFSDataPrunerTest {
}
- @Test(expected = RuntimeException.class)
+ @Test
public void testThrowsIsDirectory() throws Exception {
FileSystem testFS = mock(FileSystem.class);
- when(testFS.isDirectory((Path) any())).thenThrow(new IOException("Test Exception"));
+ when(testFS.isDirectory( any())).thenThrow(new IOException("Test Exception"));
HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///", dataPath.getAbsolutePath() + "/file-*");
pruner.fileSystem = testFS;
HDFSDataPruner.DateFileFilter filter = pruner.new DateFileFilter(pruner, true);
+ try {
+ filter.accept(new Path("foo"));
+ Assert.fail("Expected Runtime exception, but did not receive one.");
+ }
+ catch(RuntimeException e) {
- filter.accept(new Path("foo"));
-
+ }
}
@Test
- public void testIgnoresDirectoies() throws Exception {
+ public void testIgnoresDirectories() throws Exception {
FileSystem testFS = mock(FileSystem.class);
- when(testFS.isDirectory((Path) any())).thenReturn(true);
+ when(testFS.isDirectory( any())).thenReturn(true);
HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///", dataPath.getAbsolutePath() + "/file-*");
pruner.fileSystem = testFS;
@@ -127,19 +132,23 @@ public class HDFSDataPrunerTest {
}
- @Test(expected = RuntimeException.class)
+ @Test
public void testThrowBadFile() throws Exception {
FileSystem testFS = mock(FileSystem.class);
- when(testFS.isDirectory((Path) any())).thenReturn(false);
- when(testFS.getFileStatus((Path) any())).thenThrow(new IOException("Test Exception"));
+ when(testFS.isDirectory( any())).thenReturn(false);
+ when(testFS.getFileStatus( any())).thenThrow(new IOException("Test Exception"));
HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///", dataPath.getAbsolutePath() + "/file-*");
pruner.fileSystem = testFS;
HDFSDataPruner.DateFileFilter filter = pruner.new DateFileFilter(pruner, true);
-
- filter.accept(new Path("foo"));
+ try {
+ filter.accept(new Path("foo"));
+ Assert.fail("Expected Runtime exception, but did not receive one.");
+ }
+ catch(RuntimeException e) {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 974d396..90b091a 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -150,7 +150,7 @@
<version>2.12.4</version>
<configuration>
<!-- Skip the default running of this plug-in (or everything is run twice...see below) -->
- <argLine>-Xmx2048m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx2048m</argLine>
<skip>true</skip>
<!-- Show 100% of the lines from the stack trace (doesn't work) -->
<trimStackTrace>false</trimStackTrace>
@@ -195,7 +195,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
+ <version>${global_shade_version}</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml b/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
index bacaae3..1692f2e 100644
--- a/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-elasticsearch/src/main/assembly/assembly.xml
@@ -31,14 +31,15 @@
</fileSet>
<fileSet>
<directory>${project.basedir}/src/main/scripts</directory>
- <outputDirectory>/scripts</outputDirectory>
+ <outputDirectory>/bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
<exclude>**/*.formatted</exclude>
<exclude>**/*.filtered</exclude>
</excludes>
- <fileMode>0644</fileMode>
+ <fileMode>0755</fileMode>
<lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
</fileSet>
<fileSet>
<directory>${project.basedir}/target</directory>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
index ed80d82..9149d09 100755
--- a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
+++ b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-METRON_VERSION=0.1BETA
+METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
-TOPOLOGY_JAR=metron-elasticsearch-$METRON_VERSION.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/config/enrichment/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
+TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index f6244a0..756a0b4 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -178,7 +178,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
<configuration>
- <targetJdk>1.7</targetJdk>
+ <targetJdk>${global_java_version}</targetJdk>
</configuration>
</plugin>
</plugins>
@@ -190,14 +190,14 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${global_java_version}</source>
+ <target>${global_java_version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>1.4</version>
+ <version>${global_shade_version}</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<artifactSet>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index 796dcc4..fdc8f36 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -20,7 +20,7 @@
<fileSets>
<fileSet>
<directory>${project.basedir}/src/main/scripts</directory>
- <outputDirectory>/scripts</outputDirectory>
+ <outputDirectory>/bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
<exclude>**/*.formatted</exclude>
@@ -28,6 +28,7 @@
</excludes>
<fileMode>0755</fileMode>
<lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
</fileSet>
<fileSet>
<directory>${project.basedir}/src/main/flux</directory>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/28c250d1/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh b/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh
index d424ba1..ea92c61 100755
--- a/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh
+++ b/metron-platform/metron-enrichment/src/main/scripts/latency_summarizer.sh
@@ -26,7 +26,7 @@ if [ -e /usr/libexec/bigtop-detect-javahome ]; then
elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
-export METRON_VERSION=0.1BETA
+export METRON_VERSION=${project.version}
export METRON_HOME=/usr/metron/$METRON_VERSION
export TOPOLOGIES_JAR=Metron-Topologies-$METRON_VERSION.jar
java -cp $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.metron.enrichment.cli.LatencySummarizer "$@"