You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2016/09/26 14:05:04 UTC
[1/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Repository: incubator-metron
Updated Branches:
refs/heads/master 6834e146d -> 125dbef1e
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
index 05291f1..18d3b8d 100644
--- a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
@@ -48,7 +48,7 @@ configurations:
dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
- hadoop-env:
- namenode_heapsize: 1024
+ namenode_heapsize: 2048
dtnode_heapsize: 1024
- hbase-env:
hbase_regionserver_heapsize: 1024
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 8b24735..429981e 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -39,7 +39,7 @@ configurations:
dataDir: '{{ zookeeper_data_dir }}'
- hadoop-env:
hadoop_heapsize: 1024
- namenode_heapsize: 512
+ namenode_heapsize: 2048
dtnode_heapsize: 512
namenode_opt_permsize: 128m
- hbase-env:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 72b2bba..a976d00 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -42,7 +42,7 @@ configurations:
- zoo.cfg:
dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
- hadoop-env:
- namenode_heapsize: 1024
+ namenode_heapsize: 2048
dtnode_heapsize: 1024
- hbase-env:
hbase_regionserver_heapsize: 1024
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index 4358370..b412ed8 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -43,6 +43,18 @@
<filtered>true</filtered>
</fileSet>
<fileSet>
+ <directory>${project.basedir}/src/main/resources/ddl</directory>
+ <outputDirectory>/ddl</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
<directory>${project.basedir}/src/main/flux</directory>
<outputDirectory>/flux</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
index 5b1bf9b..e10af77 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
@@ -45,7 +45,7 @@ public class GeoAdapter extends JdbcAdapter {
public JSONObject enrich(CacheKey value) {
JSONObject enriched = new JSONObject();
if(!resetConnectionIfNecessary()) {
- _LOG.error("Enrichment failure, cannot maintain a connection to JDBC. Please check connection. In the meantime, I'm not enriching.");
+ _LOG.error("GEO Enrichment failure, cannot maintain a connection to JDBC. Please check connection. In the meantime, I'm not enriching.");
return enriched;
}
try {
@@ -78,9 +78,10 @@ public class GeoAdapter extends JdbcAdapter {
}
resultSet.close();
} catch (Exception e) {
- _LOG.error("Enrichment failure: " + e.getMessage(), e);
+ _LOG.error("GEO Enrichment failure: " + e.getMessage(), e);
return new JSONObject();
}
+ _LOG.trace("GEO Enrichment success: ", enriched);
return enriched;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
index d2af3cc..240531c 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
@@ -99,6 +99,7 @@ public class SimpleHBaseAdapter implements EnrichmentAdapter<CacheKey>,Serializa
throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e);
}
}
+ _LOG.trace("SimpleHBaseAdapter succeeded:", enriched);
return enriched;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
index 9fa7363..4950ddb 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
@@ -27,6 +27,8 @@ import org.apache.metron.common.stellar.StellarProcessor;
import org.apache.metron.enrichment.bolt.CacheKey;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.util.Map;
@@ -35,6 +37,7 @@ import java.util.function.Function;
import static org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.STELLAR_CONTEXT_CONF;
public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable {
+ protected static final Logger _LOG = LoggerFactory.getLogger(StellarAdapter.class);
private enum EnrichmentType implements Function<SensorEnrichmentConfig, ConfigHandler>{
ENRICHMENT(config -> config.getEnrichment().getEnrichmentConfigs().get("stellar"))
@@ -78,6 +81,7 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
Map<String, Object> globalConfig = value.getConfig().getConfiguration();
Map<String, Object> sensorConfig = value.getConfig().getEnrichment().getConfig();
if(handler == null) {
+ _LOG.trace("Stellar ConfigHandler is null.");
return new JSONObject();
}
Map<String, Object> message = value.getValue(Map.class);
@@ -102,7 +106,9 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
}
}
}
- return new JSONObject(message);
+ JSONObject enriched = new JSONObject(message);
+ _LOG.trace("Stellar Enrichment Success: " + enriched);
+ return enriched;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
index cb7ebe0..3f16b55 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
@@ -89,16 +89,17 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
String enrichmentType = enrichmentTypes.get(i++);
if (isThreat) {
enriched.put(enrichmentType, "alert");
- _LOG.trace("Enriched value => " + enriched);
+ _LOG.trace("Theat Intel Enriched value => ", enriched);
}
}
}
catch(IOException e) {
_LOG.error("Unable to retrieve value: " + e.getMessage(), e);
initializeAdapter();
- throw new RuntimeException("Unable to retrieve value", e);
+ throw new RuntimeException("Theat Intel Unable to retrieve value", e);
}
}
+ _LOG.trace("Threat Intel Enrichment Success:", enriched);
return enriched;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 2eb4b0a..e0c96a2 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -115,6 +115,7 @@ public abstract class JoinBolt<V> extends ConfiguredEnrichmentBolt {
);
cache.invalidate(key);
collector.ack(tuple);
+ LOG.trace("Emitted message for key: {}", key);
} else {
cache.put(key, streamMessageMap);
if(LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 7ff8739..bb0bee5 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -89,6 +89,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
@Override
public JSONObject joinMessages(Map<String, JSONObject> streamMessageMap) {
JSONObject ret = super.joinMessages(streamMessageMap);
+ LOG.trace("Received joined messages: {}", ret);
boolean isAlert = ret.containsKey("is_alert");
if(!isAlert) {
for (Object key : ret.keySet()) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
+ (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+ BEGIN
+ DECLARE ipn INTEGER UNSIGNED;
+ DECLARE locID_var INTEGER;
+ IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+ END IF;
+ SET ipn = INET_ATON(ip);
+ SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
+ RETURN locID_var;
+ END
+$$
+DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
index 91e8446..c4f4150 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
@@ -101,6 +101,7 @@ public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
, bulkMessageWriter
, configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
);
+ LOG.trace("Writing enrichment message: {}", message);
}
catch(Exception e) {
throw new RuntimeException("This should have been caught in the writerComponent. If you see this, file a JIRA", e);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 659a467..ce0b177 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,6 +27,7 @@
<modules>
<module>metron-analytics</module>
<module>metron-platform</module>
+ <module>metron-deployment</module>
</modules>
<properties>
<twitter>@ApacheMetron</twitter>
@@ -201,8 +202,9 @@
<exclude>metron-ui/lib/public/css/font-awesome.min.css</exclude>
<exclude>metron-ui/lib/public/font/**</exclude>
<exclude>metron-ui/node_modules/**</exclude>
- <!-- pickle file - binary format -->
+ <!-- pickle file - binary format -->
<exclude>**/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/*.p</exclude>
+ <exclude>**/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p</exclude>
</excludes>
</configuration>
</plugin>
[7/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
METRON-427 Create Ambari Management Pack for Metron Installation closes apache/incubator-metron#266
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/125dbef1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/125dbef1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/125dbef1
Branch: refs/heads/master
Commit: 125dbef1e59ff808a62e4f5a7d265aafbcf6bf08
Parents: 6834e14
Author: justinleet <ju...@gmail.com>
Authored: Mon Sep 26 09:53:37 2016 -0400
Committer: justinjleet <ju...@gmail.com>
Committed: Mon Sep 26 09:53:37 2016 -0400
----------------------------------------------------------------------
.gitignore | 1 +
metron-deployment/README.md | 63 +-
.../packaging/ambari/metron-mpack/pom.xml | 111 ++
.../src/main/assemblies/metron-mpack.xml | 43 +
.../ELASTICSEARCH/2.3.3/metainfo.xml | 29 +
.../addon-services/KIBANA/4.5.1/metainfo.xml | 33 +
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 27 +
.../METRON/0.2.0BETA/metainfo.xml | 26 +
.../METRON/0.2.0BETA/role_command_order.json | 14 +
.../2.3.3/configuration/elastic-env.xml | 58 +
.../2.3.3/configuration/elastic-site.xml | 180 ++
.../2.3.3/configuration/elastic-sysconfig.xml | 98 ++
.../ELASTICSEARCH/2.3.3/metainfo.xml | 73 +
.../2.3.3/package/scripts/elastic.py | 68 +
.../2.3.3/package/scripts/elastic_master.py | 81 +
.../2.3.3/package/scripts/elastic_slave.py | 77 +
.../2.3.3/package/scripts/params.py | 77 +
.../2.3.3/package/scripts/properties_config.py | 34 +
.../2.3.3/package/scripts/service_check.py | 80 +
.../2.3.3/package/scripts/slave.py | 65 +
.../2.3.3/package/scripts/status_params.py | 26 +
.../templates/elasticsearch.master.yaml.j2 | 84 +
.../templates/elasticsearch.slave.yaml.j2 | 84 +
.../ELASTICSEARCH/2.3.3/role_command_order.json | 8 +
.../KIBANA/4.5.1/configuration/kibana-env.xml | 54 +
.../KIBANA/4.5.1/configuration/kibana-site.xml | 112 ++
.../common-services/KIBANA/4.5.1/metainfo.xml | 75 +
.../4.5.1/package/scripts/dashboard/__init__.py | 16 +
.../4.5.1/package/scripts/dashboard/dashboard.p | 1539 ++++++++++++++++++
.../package/scripts/dashboard/dashboardindex.py | 95 ++
.../4.5.1/package/scripts/kibana_master.py | 137 ++
.../KIBANA/4.5.1/package/scripts/params.py | 47 +
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 +
.../0.2.0BETA/configuration/metron-env.xml | 174 ++
.../METRON/0.2.0BETA/metainfo.xml | 273 ++++
.../0.2.0BETA/package/files/addMysqlUser.sh | 44 +
.../0.2.0BETA/package/files/bro_index.template | 217 +++
.../0.2.0BETA/package/files/createMysqlGeoIp.sh | 42 +
.../0.2.0BETA/package/files/removeMysqlUser.sh | 33 +
.../package/files/snort_index.template | 183 +++
.../0.2.0BETA/package/files/yaf_index.template | 205 +++
.../package/scripts/enrichment_commands.py | 161 ++
.../package/scripts/enrichment_master.py | 87 +
.../package/scripts/indexing_commands.py | 142 ++
.../package/scripts/indexing_master.py | 120 ++
.../0.2.0BETA/package/scripts/metron_service.py | 76 +
.../0.2.0BETA/package/scripts/mysql_server.py | 61 +
.../0.2.0BETA/package/scripts/mysql_service.py | 46 +
.../0.2.0BETA/package/scripts/mysql_users.py | 58 +
.../0.2.0BETA/package/scripts/mysql_utils.py | 50 +
.../package/scripts/params/__init__.py | 18 +
.../0.2.0BETA/package/scripts/params/params.py | 32 +
.../package/scripts/params/params_linux.py | 169 ++
.../package/scripts/params/params_windows.py | 20 +
.../package/scripts/params/status_params.py | 57 +
.../package/scripts/parser_commands.py | 208 +++
.../0.2.0BETA/package/scripts/parser_master.py | 89 +
.../0.2.0BETA/package/scripts/service_check.py | 41 +
.../package/templates/enrichment.properties.j2 | 88 +
.../METRON/0.2.0BETA/quicklinks/quicklinks.json | 28 +
.../metron-mpack/src/main/resources/mpack.json | 85 +
.../2.3.3/configuration/elastic-env.xml | 58 -
.../2.3.3/configuration/elastic-site.xml | 180 --
.../2.3.3/configuration/elastic-sysconfig.xml | 98 --
.../ELASTICSEARCH/2.3.3/metainfo.xml | 73 -
.../2.3.3/package/scripts/elastic.py | 68 -
.../2.3.3/package/scripts/elastic_master.py | 79 -
.../2.3.3/package/scripts/elastic_slave.py | 76 -
.../2.3.3/package/scripts/params.py | 77 -
.../2.3.3/package/scripts/properties_config.py | 34 -
.../2.3.3/package/scripts/service_check.py | 80 -
.../2.3.3/package/scripts/slave.py | 65 -
.../2.3.3/package/scripts/status_params.py | 26 -
.../templates/elasticsearch.master.yaml.j2 | 84 -
.../templates/elasticsearch.slave.yaml.j2 | 84 -
.../ELASTICSEARCH/2.3.3/role_command_order.json | 8 -
.../0.2.0BETA/configuration/metron-indexing.xml | 48 -
.../INDEXING/0.2.0BETA/metainfo.xml | 105 --
.../0.2.0BETA/package/scripts/commands.py | 146 --
.../package/scripts/indexing_master.py | 71 -
.../0.2.0BETA/package/scripts/params.py | 29 -
.../0.2.0BETA/package/scripts/params_linux.py | 70 -
.../0.2.0BETA/package/scripts/params_windows.py | 17 -
.../0.2.0BETA/package/scripts/service_check.py | 37 -
.../0.2.0BETA/package/scripts/status_params.py | 19 -
.../0.2.0BETA/quicklinks/quicklinks.json | 28 -
.../KIBANA/4.5.1/configuration/kibana-env.xml | 54 -
.../KIBANA/4.5.1/configuration/kibana-site.xml | 112 --
.../common-services/KIBANA/4.5.1/metainfo.xml | 75 -
.../4.5.1/package/scripts/dashboard/__init__.py | 16 -
.../4.5.1/package/scripts/dashboard/dashboard.p | 1539 ------------------
.../package/scripts/dashboard/dashboardindex.py | 95 --
.../4.5.1/package/scripts/kibana_master.py | 146 --
.../KIBANA/4.5.1/package/scripts/params.py | 47 -
.../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 -
.../0.2.0BETA/configuration/metron-parsers.xml | 57 -
.../PARSERS/0.2.0BETA/metainfo.xml | 108 --
.../0.2.0BETA/package/scripts/commands.py | 235 ---
.../package/scripts/params/__init__.py | 18 -
.../0.2.0BETA/package/scripts/params/params.py | 32 -
.../package/scripts/params/params_linux.py | 114 --
.../package/scripts/params/params_windows.py | 20 -
.../package/scripts/params/status_params.py | 22 -
.../0.2.0BETA/package/scripts/parser_master.py | 77 -
.../0.2.0BETA/package/scripts/service_check.py | 40 -
.../package/templates/metron-global.json | 1 -
.../0.2.0BETA/quicklinks/quicklinks.json | 28 -
.../PARSERS/0.2.0BETA/service_advisor.py | 68 -
.../HDP/2.3/services/PARSERS/metainfo.xml | 27 -
.../services/PARSERS/role_command_order.json | 17 -
.../HDP/2.4/services/ELASTICSEARCH/metainfo.xml | 41 -
.../HDP/2.4/services/INDEXING/metainfo.xml | 26 -
.../services/INDEXING/role_command_order.json | 9 -
.../stacks/HDP/2.4/services/KIBANA/metainfo.xml | 29 -
.../docker/rpm-docker/SPECS/metron.spec | 2 +
.../packaging/docker/rpm-docker/pom.xml | 218 +++
metron-deployment/pom.xml | 181 +-
.../roles/ambari_common/defaults/main.yml | 2 +-
.../vars/multi_vagrant_cluster.yml | 2 +-
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
.../roles/ambari_config/vars/small_cluster.yml | 2 +-
.../src/main/assembly/assembly.xml | 12 +
.../enrichment/adapters/geo/GeoAdapter.java | 5 +-
.../simplehbase/SimpleHBaseAdapter.java | 1 +
.../adapters/stellar/StellarAdapter.java | 8 +-
.../threatintel/ThreatIntelAdapter.java | 5 +-
.../apache/metron/enrichment/bolt/JoinBolt.java | 1 +
.../enrichment/bolt/ThreatIntelJoinBolt.java | 1 +
.../src/main/resources/ddl/geoip_ddl.sql | 49 +
.../writer/bolt/BulkMessageWriterBolt.java | 1 +
pom.xml | 4 +-
131 files changed, 6671 insertions(+), 4916 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 49e1afa..5e570e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,4 @@ tmp/**/*
temp/**
temp/**/*
+repodata/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index 6a44596..eceeb80 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -3,6 +3,8 @@ This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Me
playbooks. These playbooks currently only target RHEL/CentOS 6.x operating
systems.
+In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README.
+
## Prerequisites
The following tools are required to run these scripts:
@@ -27,7 +29,7 @@ structure can be found at http://docs.ansible.com/ansible/playbooks_best_practic
## Ambari
The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This
-section can be skipped if installing Metron on a pre-existing cluster.
+section can be skipped if installing Metron on a pre-existing cluster.
Currently, this playbook supports building a local development cluster running on one node but options for other types
of clusters will be added in the future.
@@ -91,6 +93,62 @@ installed
Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. This also provides a good
example of how to run a full end-to-end Metron install.
+## Ambari Management Pack
+An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update.
+
+This will set up
+- Metron Parsers
+- Enrichment
+- Indexing
+- GeoIP database on MySQL
+- Optional Elasticsearch
+- Optional Kibana
+
+### Prerequisites
+- A cluster managed by Ambari 2.4
+- Metron RPMs available on the cluster in the /localrepo directory. See [RPM](#RPM) for further information.
+
+### Building Management Pack
+From `metron-deployment` run
+```
+mvn clean package
+```
+
+A tar.gz that can be used with Ambari can be found at `metron-deployment/packaging/ambari/metron-mpack/target/`
+
+### Installing Management Pack
+Before installing the mpack, update Storm's topology.classpath in Ambari to include '/etc/hbase/conf:/etc/hadoop/conf'. Restart Storm service.
+
+Place the mpack's tar.gz onto the node running Ambari Server. From the command line on this node, run
+```
+ambari-server install-mpack --mpack=<mpack_location> --verbose
+```
+
+This will make the services available in Ambari in the same manner as any services in a stack, e.g. through Add Services or during cluster install.
+The Indexing / Parsers/ Enrichment masters should be colocated with a Kafka Broker (to create topics) and HBase client (to create the enrichment and theatintel tables).
+This colocation is currently not enforced by Ambari, and should be managed by either a Service or Stack advisor as an enhancement.
+
+Several configuration parameters will need to be filled in, and should be pretty self explanatory (primarily a couple of Elasticsearch configs, and the Storm REST URL). Examples are provided in the descriptions on Ambari.
+Notably, the URL for the GeoIP database that is preloaded (and is prefilled by default) can be set to use a `file://` location
+
+After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates. Similar to this, a custom Kibana action to Load Template is available.
+
+#### Offline installation
+Currently there is only one point that would reach out to the internet during an install. This is the URL for the GeoIP database information that is preloaded into MySQL.
+
+The RPMs DO NOT reach out to the internet (because there is currently no hosting for them). They look on the local filesystem in `/localrepo`.
+
+### Current Limitations
+There are a set of limitations that should be addressed based to improve the current state of the mpacks.
+
+- MySQL install should be optional (and allow for using an existing instance).
+- MySQL should not be installed on a node already running a MySQL instance (e.g. an Ambari Server using MySQL as its database).
+- There is currently no hosting for RPMs remotely. They will have to be built locally.
+- Colocation of appropriate services should be enforced by Ambari. See [#Installing Management Pack] for more details.
+- Storm's topology.classpath is not updated with the Metron service install and needs to be updated separately.
+- Several configuration parameters used when installing the Metron service could (and should) be grabbed from Ambari. Install will require them to be manually entered.
+- Need to handle upgrading Metron
+
## RPM
RPMs can be built to install the components in metron-platform. These RPMs are built in a Docker container and placed into `target`.
@@ -111,7 +169,7 @@ Components in the RPMs:
### Building RPMs
From `metron-deployment` run
```
-mvn clean package
+mvn clean package -Pbuild-rpms
```
The output RPM files will land in `target/RPMS/noarch`. They can be installed with the standard
@@ -122,3 +180,4 @@ rpm -i <package>
## TODO
- migrate existing MySQL/GeoLite playbook
- Support Ubuntu deployments
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
new file mode 100644
index 0000000..0a40f75
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -0,0 +1,111 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.metron.packaging.mpacks</groupId>
+ <artifactId>metron_mpack</artifactId>
+ <version>1.0.0.0-SNAPSHOT</version>
+ <name>Metron Ambari Management Pack</name>
+
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-deployment</artifactId>
+ <version>0.2.0BETA</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+
+ <dependencies>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>parse-version</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>parse-version</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>mpackVersion</name>
+ <value>${project.version}</value>
+ <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex>
+ <replacement>$1.$2.$3.$4</replacement>
+ <failIfNoMatch>false</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>mpack.json</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/main/assemblies/metron-mpack.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
new file mode 100644
index 0000000..7de489f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
@@ -0,0 +1,43 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+ <id></id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/common-services</directory>
+ <outputDirectory>common-services</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/addon-services</directory>
+ <outputDirectory>addon-services</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <files>
+ <file>
+ <source>target/mpack.json</source>
+ </file>
+ </files>
+ <dependencySets>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
new file mode 100755
index 0000000..e2e6cdd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <version>2.3.3</version>
+ <extends>common-services/ELASTICSEARCH/2.3.3</extends>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
new file mode 100755
index 0000000..156a34a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <extends>common-services/ELASTICSEARCH/2.3.3</extends>
+ </service>
+ <service>
+ <name>KIBANA</name>
+ <extends>common-services/KIBANA/4.5.1</extends>
+ </service>
+ </services>
+</metainfo>
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
new file mode 100755
index 0000000..622a512
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json
@@ -0,0 +1,27 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "kibana_master_ui",
+ "label": "Metron Dashboard",
+ "requires_user_name": "false",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..a3c984c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <version>0.2.0BETA</version>
+ <extends>common-services/METRON/0.2.0BETA</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
new file mode 100755
index 0000000..0f4cd39
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/0.2.0BETA/role_command_order.json
@@ -0,0 +1,14 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
+ "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
+ "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+ "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
+ "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
new file mode 100755
index 0000000..2939c28
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_user</name>
+ <value>elasticsearch</value>
+ <property-type>USER</property-type>
+ <description>The user for Elasticsearch</description>
+ </property>
+ <property>
+ <name>user_group</name>
+ <value>elasticsearch</value>
+ <description>The group for Elasticsearch</description>
+ </property>
+ <property>
+ <name>elastic_log_dir</name>
+ <value>/var/log/elasticsearch</value>
+ <description>Log directory for elastic</description>
+ </property>
+ <property>
+ <name>elastic_pid_dir</name>
+ <value>/var/run/elasticsearch</value>
+ <description>The directory for pid files</description>
+ </property>
+
+ <!-- elasticsearch-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic-env.sh file</description>
+ <value>
+#!/bin/bash
+
+# Set ELASTICSEARCH specific environment variables here.
+
+# The java implementation to use.
+export JAVA_HOME={{java64_home}}
+export PATH=$PATH:$JAVA_HOME/bin
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
new file mode 100755
index 0000000..fb3a443
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Elastic search Configurations -->
+
+<configuration supports_final="true">
+ <!-- Configurations -->
+ <property>
+ <name>cluster_name</name>
+ <value>metron</value>
+ <description>Cluster name identifies your cluster</description>
+ </property>
+ <property>
+ <name>zen_discovery_ping_unicast_hosts</name>
+ <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
+ <value></value>
+ <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description>
+ </property>
+ <property>
+ <name>index_number_of_shards</name>
+ <value>4</value>
+ <description>Set the number of shards (splits) of an index</description>
+ </property>
+ <property>
+ <name>index_number_of_replicas</name>
+ <value>2</value>
+ <description>Set the number of replicas (additional copies) of an index</description>
+ </property>
+ <!-- Logging Configurations -->
+ <property>
+ <name>path_data</name>
+ <value>"/opt/lmm/es_data"</value>
+ <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description>
+ </property>
+ <!-- Discovery -->
+ <property>
+ <name>transport_tcp_port</name>
+ <value>9300-9400</value>
+ <description>Set a custom port for the node to node communication</description>
+ </property>
+ <property>
+ <name>http_port</name>
+ <value>9200-9300</value>
+ <description>Set a custom port to listen for HTTP traffic</description>
+ </property>
+ <!-- Discovery -->
+ <property>
+ <name>discovery_zen_ping_multicast_enabled</name>
+ <value>false</value>
+ <description>master eligible nodes</description>
+ </property>
+ <property>
+ <name>discovery_zen_ping_timeout</name>
+ <value>3s</value>
+ <description>Wait for ping responses for master discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_interval</name>
+ <value>15s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_timeout</name>
+ <value>60s</value>
+ <description>Wait for ping for cluster discovery</description>
+ </property>
+ <property>
+ <name>discovery_zen_fd_ping_retries</name>
+ <value>5</value>
+ <description>Number of ping retries before blacklisting</description>
+ </property>
+ <!-- Gateway -->
+ <property>
+ <name>gateway_recover_after_data_nodes</name>
+ <value>3</value>
+ <description>Recover as long as this many data or master nodes have joined the cluster.</description>
+ </property>
+ <property>
+ <name>recover_after_time</name>
+ <value>15m</value>
+ <description>recover_after_time</description>
+ </property>
+ <property>
+ <name>expected_data_nodes</name>
+ <value>0</value>
+ <description>expected_data_nodes</description>
+ </property>
+ <!-- Index -->
+ <property>
+ <name>index_merge_scheduler_max_thread_count</name>
+ <value>5</value>
+ <description>index.merge.scheduler.max_thread_count</description>
+ </property>
+ <property>
+ <name>indices_memory_index_store_throttle_type</name>
+ <value>none</value>
+ <description>index_store_throttle_type</description>
+ </property>
+ <property>
+ <name>index_refresh_interval</name>
+ <value>1s</value>
+ <description>index refresh interval</description>
+ </property>
+ <property>
+ <name>index_translog_flush_threshold_size</name>
+ <value>5g</value>
+ <description>index_translog_flush_threshold_size</description>
+ </property>
+ <property>
+ <name>indices_memory_index_buffer_size</name>
+ <value>10%</value>
+ <description>Percentage of heap used for write buffers</description>
+ </property>
+ <property>
+ <name>bootstrap_mlockall</name>
+ <value>true</value>
+ <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
+ </property>
+ <property>
+ <name>threadpool_bulk_queue_size</name>
+ <value>3000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
+ </property>
+ <property>
+ <name>threadpool_index_queue_size</name>
+ <value>1000</value>
+ <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
+ </property>
+ <property>
+ <name>indices_cluster_send_refresh_mapping</name>
+ <value>false</value>
+ <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
+ </property>
+ <property>
+ <name>indices_fielddata_cache_size</name>
+ <value>25%</value>
+ <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_high</name>
+ <value>0.99</value>
+ <description>Property used when multiple drives are used to understand max thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_threshold_enabled</name>
+ <value>true</value>
+ <description>Property used when multiple drives are used to understand if thresholding is active</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_disk_watermark_low</name>
+ <value>.97</value>
+ <description>Property used when multiple drives are used to understand min thresholds</description>
+ </property>
+ <property>
+ <name>cluster_routing_allocation_node_concurrent_recoveries</name>
+ <value>4</value>
+ <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
+ </property>
+ <property>
+ <name>network_host</name>
+ <value>_lo_,_eth0_</value>
+ <description>Network interface(s) will bind to. </description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
new file mode 100755
index 0000000..58e4916
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>elastic_home</name>
+ <value>/usr/share/elasticsearch/</value>
+ <description>Elasticsearch Home Directory</description>
+ </property>
+ <property>
+ <name>data_dir</name>
+ <value>/var/lib/elasticsearch/</value>
+ <description>Elasticsearch Data Directory</description>
+ </property>
+ <property>
+ <name>work_dir</name>
+ <value>/tmp/elasticsearch/</value>
+ <description>Elasticsearch Work Directory</description>
+ </property>
+ <property>
+ <name>conf_dir</name>
+ <value>/etc/elasticsearch/</value>
+ <description>Elasticsearch Configuration Directory</description>
+ </property>
+ <property>
+ <name>heap_size</name>
+ <value>128m</value>
+ <description>Heap size</description>
+ </property>
+ <property>
+ <name>max_open_files</name>
+ <value>65535</value>
+ <description>Maximum number of open files</description>
+ </property>
+ <property>
+ <name>max_map_count</name>
+ <value>262144</value>
+ <description>Maximum number of memory map areas for process</description>
+ </property>
+
+ <!--/etc/sysconfig/elasticsearch-->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for elastic-env.sh file</description>
+ <value>
+# Directory where the Elasticsearch binary distribution resides
+ES_HOME={{elastic_home}}
+
+# Heap Size (defaults to 256m min, 1g max)
+ES_HEAP_SIZE={{heap_size}}
+
+# Maximum number of open files
+MAX_OPEN_FILES={{max_open_files}}
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+MAX_MAP_COUNT={{max_map_count}}
+
+# Elasticsearch log directory
+LOG_DIR={{log_dir}}
+
+# Elasticsearch data directory
+DATA_DIR={{data_dir}}
+
+# Elasticsearch work directory
+WORK_DIR={{work_dir}}
+
+# Elasticsearch conf directory
+CONF_DIR={{conf_dir}}
+
+# User to run as, change this to a specific elasticsearch user if possible
+# Also make sure, this user can write into the log directories in case you change them
+# This setting only works for the init script, but has to be configured separately for systemd startup
+ES_USER={{elastic_user}}
+
+# Additional Java OPTS
+ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
+-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
+-XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8"
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
new file mode 100755
index 0000000..25f98e5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ELASTICSEARCH</name>
+ <displayName>Elasticsearch</displayName>
+ <comment>Indexing and Search</comment>
+ <version>2.3.3</version>
+ <components>
+ <component>
+ <name>ES_MASTER</name>
+ <displayName>Elasticsearch Master</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>ES_SLAVE</name>
+ <displayName>Elasticsearch Data Node</displayName>
+ <category>SLAVE</category>
+ <cardinality>3+</cardinality>
+ <commandScript>
+ <script>scripts/elastic_slave.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch-2.3.3</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>elastic-env</config-type>
+ <config-type>elastic-site</config-type>
+ <config-type>elastic-sysconfig</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
new file mode 100755
index 0000000..a472655
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+
+
+def elastic():
+ print "INSIDE THE %s" % __file__
+ import params
+
+ params.path_data = params.path_data.replace('"', '')
+ data_path = params.path_data.replace(' ', '').split(',')
+ data_path[:] = [x.replace('"', '') for x in data_path]
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ directories = directories + data_path
+
+ Directory(directories,
+ create_parents=True,
+ # recursive=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir)
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ content=InlineTemplate(params.elastic_env_sh_template)
+ )
+
+ configurations = params.config['configurations']['elastic-site']
+
+ print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir)
+ File("{0}/elasticsearch.yml".format(params.conf_dir),
+ content=Template(
+ "elasticsearch.master.yaml.j2",
+ configurations=configurations),
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master sysconfig: /etc/sysconfig/elasticsearch"
+ File(format("/etc/sysconfig/elasticsearch"),
+ owner="root",
+ group="root",
+ content=InlineTemplate(params.sysconfig_template)
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
new file mode 100755
index 0000000..3ad4e1f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
@@ -0,0 +1,81 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+from elastic import elastic
+
+
+class Elasticsearch(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ print 'Install the Master'
+ Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
+ Execute("echo \"[elasticsearch-2.x]\n"
+ "name=Elasticsearch repository for 2.x packages\n"
+ "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
+ "gpgcheck=1\n"
+ "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
+ "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
+
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+
+ elastic()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ stop_cmd = format("service elasticsearch stop")
+ print 'Stop the Master'
+ Execute(stop_cmd)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+ start_cmd = format("service elasticsearch start")
+ print 'Start the Master'
+ Execute(start_cmd)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ status_cmd = format("service elasticsearch status")
+ print 'Status of the Master'
+ Execute(status_cmd)
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ restart_cmd = format("service elasticsearch restart")
+ print 'Restarting the Master'
+ Execute(restart_cmd)
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
new file mode 100755
index 0000000..4b636e8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
@@ -0,0 +1,77 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+from slave import slave
+
+
+class Elasticsearch(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ print 'Install the Slave'
+ Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
+ Execute("echo \"[elasticsearch-2.x]\n"
+ "name=Elasticsearch repository for 2.x packages\n"
+ "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
+ "gpgcheck=1\n"
+ "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
+ "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+ slave()
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ stop_cmd = format("service elasticsearch stop")
+ print 'Stop the Slave'
+ Execute(stop_cmd)
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ start_cmd = format("service elasticsearch start")
+ print 'Start the Slave'
+ Execute(start_cmd)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ status_cmd = format("service elasticsearch status")
+ print 'Status of the Slave'
+ Execute(status_cmd)
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ restart_cmd = format("service elasticsearch restart")
+ print 'Restarting the Slave'
+ Execute(restart_cmd)
+
+
+if __name__ == "__main__":
+ Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
new file mode 100755
index 0000000..30e88ea
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
+data_dir = config['configurations']['elastic-sysconfig']['data_dir']
+work_dir = config['configurations']['elastic-sysconfig']['work_dir']
+conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
+heap_size = config['configurations']['elastic-sysconfig']['heap_size']
+max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
+max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
+
+elastic_user = config['configurations']['elastic-env']['elastic_user']
+user_group = config['configurations']['elastic-env']['user_group']
+log_dir = config['configurations']['elastic-env']['elastic_log_dir']
+pid_dir = '/var/run/elasticsearch'
+pid_file = '/var/run/elasticsearch/elasticsearch.pid'
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+elastic_env_sh_template = config['configurations']['elastic-env']['content']
+sysconfig_template = config['configurations']['elastic-sysconfig']['content']
+
+cluster_name = config['configurations']['elastic-site']['cluster_name']
+zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
+
+path_data = config['configurations']['elastic-site']['path_data']
+http_port = config['configurations']['elastic-site']['http_port']
+transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
+
+recover_after_time = config['configurations']['elastic-site']['recover_after_time']
+gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
+expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
+discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled']
+index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
+index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
+index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
+indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
+index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
+index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
+indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
+bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall']
+threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
+cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
+cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
+cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']
+cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
+indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
+indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']
+threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
+
+discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
+discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
+discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
+discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
+
+network_host = config['configurations']['elastic-site']['network_host']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
new file mode 100755
index 0000000..ef9f6dd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+
+
+def properties_inline_template(configurations):
+ return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
+{% endfor %}''', configurations_dict=configurations)
+
+
+def properties_config(filename, configurations=None, conf_dir=None,
+ mode=None, owner=None, group=None, brokerid=None):
+ config_content = properties_inline_template(configurations)
+ File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
+ group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
new file mode 100755
index 0000000..e84fb01
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+import subprocess
+import sys
+
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.script import Script
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ doc = '{"name": "Ambari Smoke test"}'
+ index = "ambari_smoke_test"
+
+ print("Running Elastic search service check", file=sys.stdout)
+
+ # Make sure the service is actually up. We can live without everything allocated.
+ # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
+ # also hit ES before things are green.
+ host = "localhost:9200"
+ Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
+ logoutput=True,
+ tries=6,
+ try_sleep=20
+ )
+
+ # Put a document into a new index.
+
+ Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True)
+
+ # Retrieve the document. Use subprocess because we actually need the results here.
+ cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index)
+ proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_retrieve = stdout
+ print("Retrieval response is: %s" % response_retrieve)
+ expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
+ % (index, doc)
+
+ # Delete the index
+ cmd_delete = "curl -XDELETE '%s/%s'" % (host, index)
+ proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+ response_delete = stdout
+ print("Delete index response is: %s" % response_retrieve)
+ expected_delete = '{"acknowledged":true}'
+
+ if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
+ print("Smoke test able to communicate with Elasticsearch")
+ else:
+ print("Elasticsearch service unable to retrieve document.")
+ sys.exit(1)
+
+ exit(0)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
new file mode 100755
index 0000000..90d4d3d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+
+
+def slave():
+ import params
+
+ params.path_data = params.path_data.replace('"', '')
+ data_path = params.path_data.replace(' ', '').split(',')
+ data_path[:] = [x.replace('"', '') for x in data_path]
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ directories = directories + data_path
+
+ Directory(directories,
+ create_parents=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_user,
+ cd_access="a"
+ )
+
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ content=InlineTemplate(params.elastic_env_sh_template)
+ )
+
+ configurations = params.config['configurations']['elastic-site']
+
+ File("{0}/elasticsearch.yml".format(params.conf_dir),
+ content=Template(
+ "elasticsearch.slave.yaml.j2",
+ configurations=configurations),
+ owner=params.elastic_user,
+ group=params.elastic_user
+ )
+
+ print "Master sysconfig: /etc/sysconfig/elasticsearch"
+ File(format("/etc/sysconfig/elasticsearch"),
+ owner="root",
+ group="root",
+ content=InlineTemplate(params.sysconfig_template)
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
new file mode 100755
index 0000000..9cfb5cf
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
new file mode 100755
index 0000000..a9de018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
@@ -0,0 +1,84 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ multicast:
+ enabled: {{discovery_zen_ping_multicast_enabled}}
+ unicast:
+ hosts: "{{zen_discovery_ping_unicast_hosts}}"
+
+node:
+ data: false
+ master: true
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http.cors.enabled: true
+
+port: {{http_port}}
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+index:
+ number_of_shards: {{index_number_of_shards}}
+ merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
+ translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
+ refresh_interval: {{index_refresh_interval}}
+ number_of_replicas: {{index_number_of_replicas}}
+
+indices:
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+ cluster:
+ send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
+
+bootstrap.mlockall: {{bootstrap_mlockall}}
+
+threadpool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
new file mode 100755
index 0000000..e88fc5f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
@@ -0,0 +1,84 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+cluster:
+ name: {{cluster_name}}
+ routing:
+ allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
+ allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
+ allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
+ allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
+
+discovery:
+ zen:
+ ping:
+ multicast:
+ enabled: {{discovery_zen_ping_multicast_enabled}}
+ unicast:
+ hosts: "{{zen_discovery_ping_unicast_hosts}}"
+
+node:
+ data: true
+ master: false
+ name: {{hostname}}
+path:
+ data: {{path_data}}
+
+http.cors.enabled: true
+
+port: {{http_port}}
+
+transport:
+ tcp:
+ port: {{transport_tcp_port}}
+
+gateway:
+ recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
+ recover_after_time: {{recover_after_time}}
+ expected_data_nodes: {{expected_data_nodes}}
+
+index:
+ number_of_shards: {{index_number_of_shards}}
+ merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
+ translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
+ refresh_interval: {{index_refresh_interval}}
+ number_of_replicas: {{index_number_of_replicas}}
+
+indices:
+ memory:
+ index_buffer_size: {{indices_memory_index_buffer_size}}
+ store.throttle.type: {{indices_memory_index_store_throttle_type}}
+ fielddata:
+ cache.size: {{indices_fielddata_cache_size}}
+ cluster:
+ send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
+
+bootstrap.mlockall: {{bootstrap_mlockall}}
+
+threadpool:
+ bulk:
+ queue_size: {{threadpool_bulk_queue_size}}
+ index:
+ queue_size: {{threadpool_index_queue_size}}
+
+discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
+discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
+discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
+discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
+
+network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
new file mode 100755
index 0000000..130d018
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
@@ -0,0 +1,8 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
new file mode 100755
index 0000000..972945d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>kibana_user</name>
+ <value>kibana</value>
+ <property-type>USER</property-type>
+ <description></description>
+ </property>
+ <property>
+ <name>user_group</name>
+ <value>kibana</value>
+ <property-type>GROUP</property-type>
+ <description></description>
+ </property>
+ <property require-input="true">
+ <name>kibana_log_dir</name>
+ <value>/var/log/kibana</value>
+ <description>Log directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_pid_dir</name>
+ <value>/var/run/kibana</value>
+ <description>PID directory for Kibana</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_es_url</name>
+ <value></value>
+ <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
+ </property>
+ <property require-input="true">
+ <name>kibana_server_port</name>
+ <value>5000</value>
+ <description>Kibana back end server port to use.</description>
+ </property>
+</configuration>
[3/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
deleted file mode 100755
index a9a86c0..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-
-
-# Wrap major operations and functionality in this class
-class Commands:
- __params = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__configured = os.path.isfile(self.__params.configured_flag_file)
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{}/kafka-topics.sh \
- --zookeeper {} \
- --create \
- --topic {} \
- --partitions {} \
- --replication-factor {} \
- --config retention.bytes={}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = 10
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating topics for indexing")
-
- Logger.info("Creating topic'{}'".format(self.__params.metron_indexing_topology))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- self.__params.metron_indexing_topology,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def start_indexing_topology(self):
- Logger.info("Starting Metron indexing topology: {}".format(self.__params.metron_indexing_topology))
- start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \
- -s {} \
- -z {}"""
- Logger.info('Starting ' + self.__params.metron_indexing_topology)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.metron_indexing_topology, self.__params.zookeeper_quorum))
-
- Logger.info('Finished starting indexing topology')
-
- def stop_indexing_topology(self):
- Logger.info('Stopping ' + self.__params.metron_indexing_topology)
- stop_cmd = 'storm kill ' + self.__params.metron_indexing_topology
- Execute(stop_cmd)
- Logger.info('Done stopping indexing topologies')
-
- def restart_indexing_topology(self):
- Logger.info('Restarting the indexing topologies')
- self.stop_indexing_topology()
-
- # Wait for old topology to be cleaned up by Storm, before starting again.
- retries = 0
- topology_active = self.is_topology_active()
- while topology_active and retries < 3:
- Logger.info('Existing topology still active. Will wait and retry')
- time.sleep(40)
- topology_active = self.is_topology_active()
- retries += 1
-
- if not topology_active:
- self.start_indexing_topology()
- Logger.info('Done restarting the indexing topologies')
- else:
- Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
-
- def is_topology_active(self):
- cmd_retrieve = "storm list | grep 'indexing'"
-
- proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- Logger.info("Retrieval response is: %s" % stdout)
- Logger.warning("Error response is: %s" % stderr)
-
- fields = stdout.split()
- if len(fields) < 2:
- Logger.warning("Indexing topology is not running")
- return False
-
- # Get the second column, which is status. We already know first column is indexing)
- status = stdout.split()[1]
- running_status_set = ['ACTIVE', 'REBALANCING']
- return status in running_status_set
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
deleted file mode 100755
index bfae19a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
+++ /dev/null
@@ -1,71 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class Indexing(Script):
- def install(self, env):
- import params
- env.set_params(params)
- commands = Commands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- commands = Commands(params)
- if not commands.is_configured():
- commands.init_kafka_topics()
- commands.set_configured()
-
- commands.start_indexing_topology()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- commands = Commands(params)
- commands.stop_indexing_topology()
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- commands = Commands(status_params)
-
- if not commands.is_topology_active():
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- import params
- env.set_params(params)
- commands = Commands(params)
- commands.restart_indexing_topology()
-
- def kafkabuild(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- commands = Commands(params)
- commands.init_kafka_topics()
-
-
-if __name__ == "__main__":
- Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
deleted file mode 100755
index 6440005..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- pass
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
deleted file mode 100755
index 86a0359..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import os
-
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.script import Script
-
-# Server configurations
-config = Script.get_config()
-
-hostname = config['hostname']
-metron_home = config['configurations']['metron-indexing']['metron_home']
-metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology']
-yum_repo_type = 'local'
-metron_user = config['configurations']['metron-indexing']['metron_user']
-
-metron_config_path = metron_home + '/config'
-configured_flag_file = metron_config_path + '/metron_indexing_is_configured'
-
-# Hadoop params
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-
-# Zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Kafka
-stack_root = Script.get_stack_root()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention']
-
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
deleted file mode 100755
index b5828d2..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
deleted file mode 100755
index c02d695..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- commands = Commands(params)
- if commands.is_topology_active():
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
deleted file mode 100755
index 30aed60..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-metron_indexing_topology = 'indexing'
-configured_flag_file = ""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
deleted file mode 100755
index 972945d..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>kibana_user</name>
- <value>kibana</value>
- <property-type>USER</property-type>
- <description></description>
- </property>
- <property>
- <name>user_group</name>
- <value>kibana</value>
- <property-type>GROUP</property-type>
- <description></description>
- </property>
- <property require-input="true">
- <name>kibana_log_dir</name>
- <value>/var/log/kibana</value>
- <description>Log directory for Kibana</description>
- </property>
- <property require-input="true">
- <name>kibana_pid_dir</name>
- <value>/var/run/kibana</value>
- <description>PID directory for Kibana</description>
- </property>
- <property require-input="true">
- <name>kibana_es_url</name>
- <value></value>
- <description>The Elasticsearch instance to use for all your queries. (http://eshost:9200)</description>
- </property>
- <property require-input="true">
- <name>kibana_server_port</name>
- <value>5000</value>
- <description>Kibana back end server port to use.</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
deleted file mode 100755
index c546e2c..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <!-- kibana.yml -->
- <property>
- <name>content</name>
- <display-name>kibana.yml template</display-name>
- <description>This is the jinja template for kibana.yml file</description>
- <value>
-# Kibana is served by a back end server. This controls which port to use.
-server.port: {{ kibana_port }}
-
-# The host to bind the server to.
-# server.host: "0.0.0.0"
-
-# If you are running kibana behind a proxy, and want to mount it at a path,
-# specify that path here. The basePath can't end in a slash.
-# server.basePath: ""
-
-# The maximum payload size in bytes on incoming server requests.
-# server.maxPayloadBytes: 1048576
-
-# The Elasticsearch instance to use for all your queries.
-elasticsearch.url: {{ es_url }}
-
-# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
-# then the host you use to connect to *this* Kibana instance will be sent.
-# elasticsearch.preserveHost: true
-
-# Kibana uses an index in Elasticsearch to store saved searches, visualizations
-# and dashboards. It will create a new index if it doesn't already exist.
-# kibana.index: ".kibana"
-
-# The default application to load.
-# kibana.defaultAppId: "discover"
-
-# If your Elasticsearch is protected with basic auth, these are the user credentials
-# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
-# users will still need to authenticate with Elasticsearch (which is proxied through
-# the Kibana server)
-# elasticsearch.username: "user"
-# elasticsearch.password: "pass"
-
-# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
-# server.ssl.cert: /path/to/your/server.crt
-# server.ssl.key: /path/to/your/server.key
-
-# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
-# elasticsearch.ssl.cert: /path/to/your/client.crt
-# elasticsearch.ssl.key: /path/to/your/client.key
-
-# If you need to provide a CA certificate for your Elasticsearch instance, put
-# the path of the pem file here.
-# elasticsearch.ssl.ca: /path/to/your/CA.pem
-
-# Set to false to have a complete disregard for the validity of the SSL
-# certificate.
-# elasticsearch.ssl.verify: true
-
-# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
-# request_timeout setting
-# elasticsearch.pingTimeout: 1500
-
-# Time in milliseconds to wait for responses from the back end or elasticsearch.
-# This must be > 0
-# elasticsearch.requestTimeout: 30000
-
-# Time in milliseconds for Elasticsearch to wait for responses from shards.
-# Set to 0 to disable.
-# elasticsearch.shardTimeout: 0
-
-# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
-# elasticsearch.startupTimeout: 5000
-
-# Set the path to where you would like the process id file to be created.
-# pid.file: /var/run/kibana.pid
-
-# If you would like to send the log output to a file you can set the path below.
-logging.dest: {{ log_dir }}/kibana.log
-
-# Set this to true to suppress all logging output.
-# logging.silent: false
-
-# Set this to true to suppress all logging output except for error messages.
-# logging.quiet: false
-
-# Set this to true to log all events, including system usage information and all requests.
-# logging.verbose: false
- </value>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
deleted file mode 100755
index d14afbf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <displayName>Kibana</displayName>
- <comment>Kibana Dashboard</comment>
- <version>4.5.1</version>
- <components>
- <component>
- <name>KIBANA_MASTER</name>
- <displayName>Kibana Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <commandScript>
- <script>scripts/kibana_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>LOADTEMPLATE</name>
- <background>false</background>
- <commandScript>
- <script>scripts/kibana_master.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>python-elasticsearch</name>
- </package>
- <package>
- <name>kibana-4.5.1</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <configuration-dependencies>
- <config-type>kibana-env</config-type>
- <config-type>kibana-site</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
deleted file mode 100755
index 8d2bad8..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
deleted file mode 100755
index 8327eb8..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
+++ /dev/null
@@ -1,1539 +0,0 @@
-(lp1
-(dp2
-V_score
-p3
-F1
-sV_type
-p4
-Vvisualization
-p5
-sV_id
-p6
-VWeb-Request-Type
-p7
-sV_source
-p8
-(dp9
-VvisState
-p10
-V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p11
-sVdescription
-p12
-V
-sVtitle
-p13
-VWeb Request Type
-p14
-sVuiStateJSON
-p15
-V{}
-p16
-sVversion
-p17
-I1
-sVsavedSearchId
-p18
-Vweb-search
-p19
-sVkibanaSavedObjectMeta
-p20
-(dp21
-VsearchSourceJSON
-p22
-V{"filter":[]}
-p23
-sssV_index
-p24
-V.kibana
-p25
-sa(dp26
-V_score
-p27
-F1
-sV_type
-p28
-Vvisualization
-p29
-sV_id
-p30
-VTop-Snort-Alerts-by-Source
-p31
-sV_source
-p32
-(dp33
-VvisState
-p34
-V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}}
-p35
-sVdescription
-p36
-V
-sVtitle
-p37
-VTop Snort Alerts by Source
-p38
-sVuiStateJSON
-p39
-V{}
-p40
-sVversion
-p41
-I1
-sVkibanaSavedObjectMeta
-p42
-(dp43
-VsearchSourceJSON
-p44
-V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p45
-sssV_index
-p46
-V.kibana
-p47
-sa(dp48
-V_score
-p49
-F1
-sV_type
-p50
-Vvisualization
-p51
-sV_id
-p52
-VWelcome
-p53
-sV_source
-p54
-(dp55
-VvisState
-p56
-V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed
by Apache Metron."},"aggs":[],"listeners":{}}
-p57
-sVdescription
-p58
-V
-sVtitle
-p59
-VWelcome to Apache Metron
-p60
-sVuiStateJSON
-p61
-V{}
-p62
-sVversion
-p63
-I1
-sVkibanaSavedObjectMeta
-p64
-(dp65
-VsearchSourceJSON
-p66
-V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
-p67
-sssV_index
-p68
-V.kibana
-p69
-sa(dp70
-V_score
-p71
-F1
-sV_type
-p72
-Vsearch
-p73
-sV_id
-p74
-Vsnort-search
-p75
-sV_source
-p76
-(dp77
-Vsort
-p78
-(lp79
-Vtimestamp
-p80
-aVdesc
-p81
-asVhits
-p82
-I0
-sVdescription
-p83
-V
-sVtitle
-p84
-VSnort Alerts
-p85
-sVversion
-p86
-I1
-sVkibanaSavedObjectMeta
-p87
-(dp88
-VsearchSourceJSON
-p89
-V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
-p90
-ssVcolumns
-p91
-(lp92
-Vmsg
-p93
-aVsig_id
-p94
-aVip_src_addr
-p95
-aVip_src_port
-p96
-aVip_dst_addr
-p97
-aVip_dst_port
-p98
-assV_index
-p99
-V.kibana
-p100
-sa(dp101
-V_score
-p102
-F1
-sV_type
-p103
-Vsearch
-p104
-sV_id
-p105
-Vyaf-search
-p106
-sV_source
-p107
-(dp108
-Vsort
-p109
-(lp110
-Vtimestamp
-p111
-aVdesc
-p112
-asVhits
-p113
-I0
-sVdescription
-p114
-V
-sVtitle
-p115
-VYAF
-p116
-sVversion
-p117
-I1
-sVkibanaSavedObjectMeta
-p118
-(dp119
-VsearchSourceJSON
-p120
-V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}}
-p121
-ssVcolumns
-p122
-(lp123
-Vip_src_addr
-p124
-aVip_src_port
-p125
-aVip_dst_addr
-p126
-aVip_dst_port
-p127
-aVprotocol
-p128
-aVduration
-p129
-aVpkt
-p130
-assV_index
-p131
-V.kibana
-p132
-sa(dp133
-V_score
-p134
-F1
-sV_type
-p135
-Vconfig
-p136
-sV_id
-p137
-V4.5.1
-p138
-sV_source
-p139
-(dp140
-VbuildNum
-p141
-I9892
-sVdefaultIndex
-p142
-Vbro*
-p143
-ssV_index
-p144
-V.kibana
-p145
-sa(dp146
-V_score
-p147
-F1
-sV_type
-p148
-Vindex-pattern
-p149
-sV_id
-p150
-Vbro*
-p151
-sV_source
-p152
-(dp153
-Vfields
-p154
-V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"prot
ocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"s
cripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false
,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,
"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"ana
lyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enr
ichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","coun
t":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":
false}]
-p155
-sVtimeFieldName
-p156
-Vtimestamp
-p157
-sVtitle
-p158
-Vbro*
-p159
-ssV_index
-p160
-V.kibana
-p161
-sa(dp162
-V_score
-p163
-F1
-sV_type
-p164
-Vvisualization
-p165
-sV_id
-p166
-VFlow-Duration
-p167
-sV_source
-p168
-(dp169
-VvisState
-p170
-V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}}
-p171
-sVdescription
-p172
-V
-sVtitle
-p173
-VFlow Duration
-p174
-sVuiStateJSON
-p175
-V{"vis":{"legendOpen":false}}
-p176
-sVversion
-p177
-I1
-sVkibanaSavedObjectMeta
-p178
-(dp179
-VsearchSourceJSON
-p180
-V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p181
-sssV_index
-p182
-V.kibana
-p183
-sa(dp184
-V_score
-p185
-F1
-sV_type
-p186
-Vvisualization
-p187
-sV_id
-p188
-VEvents
-p189
-sV_source
-p190
-(dp191
-VvisState
-p192
-V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p193
-sVdescription
-p194
-V
-sVtitle
-p195
-VEvents
-p196
-sVuiStateJSON
-p197
-V{"vis":{"legendOpen":false}}
-p198
-sVversion
-p199
-I1
-sVkibanaSavedObjectMeta
-p200
-(dp201
-VsearchSourceJSON
-p202
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p203
-sssV_index
-p204
-V.kibana
-p205
-sa(dp206
-V_score
-p207
-F1
-sV_type
-p208
-Vvisualization
-p209
-sV_id
-p210
-VWeb-Request-Header
-p211
-sV_source
-p212
-(dp213
-VvisState
-p214
-V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}}
-p215
-sVdescription
-p216
-V
-sVtitle
-p217
-VWeb Request Header
-p218
-sVuiStateJSON
-p219
-V{}
-p220
-sVversion
-p221
-I1
-sVkibanaSavedObjectMeta
-p222
-(dp223
-VsearchSourceJSON
-p224
-V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p225
-sssV_index
-p226
-V.kibana
-p227
-sa(dp228
-V_score
-p229
-F1
-sV_type
-p230
-Vvisualization
-p231
-sV_id
-p232
-VTop-Alerts-By-Host
-p233
-sV_source
-p234
-(dp235
-VvisState
-p236
-V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}}
-p237
-sVdescription
-p238
-V
-sVtitle
-p239
-VTop Alerts By Host
-p240
-sVuiStateJSON
-p241
-V{}
-p242
-sVversion
-p243
-I1
-sVsavedSearchId
-p244
-Vsnort-search
-p245
-sVkibanaSavedObjectMeta
-p246
-(dp247
-VsearchSourceJSON
-p248
-V{"filter":[]}
-p249
-sssV_index
-p250
-V.kibana
-p251
-sa(dp252
-V_score
-p253
-F1
-sV_type
-p254
-Vvisualization
-p255
-sV_id
-p256
-VYAF-Flow(s)
-p257
-sV_source
-p258
-(dp259
-VvisState
-p260
-V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
-p261
-sVdescription
-p262
-V
-sVtitle
-p263
-VYAF Flows
-p264
-sVuiStateJSON
-p265
-V{}
-p266
-sVversion
-p267
-I1
-sVkibanaSavedObjectMeta
-p268
-(dp269
-VsearchSourceJSON
-p270
-V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p271
-sssV_index
-p272
-V.kibana
-p273
-sa(dp274
-V_score
-p275
-F1
-sV_type
-p276
-Vvisualization
-p277
-sV_id
-p278
-VTop-DNS-Query
-p279
-sV_source
-p280
-(dp281
-VvisState
-p282
-V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p283
-sVdescription
-p284
-V
-sVtitle
-p285
-VTop DNS Query
-p286
-sVuiStateJSON
-p287
-V{}
-p288
-sVversion
-p289
-I1
-sVkibanaSavedObjectMeta
-p290
-(dp291
-VsearchSourceJSON
-p292
-V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p293
-sssV_index
-p294
-V.kibana
-p295
-sa(dp296
-V_score
-p297
-F1
-sV_type
-p298
-Vvisualization
-p299
-sV_id
-p300
-VTotal-Events
-p301
-sV_source
-p302
-(dp303
-VvisState
-p304
-V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}}
-p305
-sVdescription
-p306
-V
-sVtitle
-p307
-VEvent Count
-p308
-sVuiStateJSON
-p309
-V{}
-p310
-sVversion
-p311
-I1
-sVkibanaSavedObjectMeta
-p312
-(dp313
-VsearchSourceJSON
-p314
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p315
-sssV_index
-p316
-V.kibana
-p317
-sa(dp318
-V_score
-p319
-F1
-sV_type
-p320
-Vvisualization
-p321
-sV_id
-p322
-VEvent-Types
-p323
-sV_source
-p324
-(dp325
-VvisState
-p326
-V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p327
-sVdescription
-p328
-V
-sVtitle
-p329
-VEvent Sources
-p330
-sVuiStateJSON
-p331
-V{}
-p332
-sVversion
-p333
-I1
-sVkibanaSavedObjectMeta
-p334
-(dp335
-VsearchSourceJSON
-p336
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p337
-sssV_index
-p338
-V.kibana
-p339
-sa(dp340
-V_score
-p341
-F1
-sV_type
-p342
-Vvisualization
-p343
-sV_id
-p344
-VUnique-Location(s)
-p345
-sV_source
-p346
-(dp347
-VvisState
-p348
-V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}}
-p349
-sVdescription
-p350
-V
-sVtitle
-p351
-VGeo-IP Locations
-p352
-sVuiStateJSON
-p353
-V{}
-p354
-sVversion
-p355
-I1
-sVkibanaSavedObjectMeta
-p356
-(dp357
-VsearchSourceJSON
-p358
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p359
-sssV_index
-p360
-V.kibana
-p361
-sa(dp362
-V_score
-p363
-F1
-sV_type
-p364
-Vvisualization
-p365
-sV_id
-p366
-VSnort-Header
-p367
-sV_source
-p368
-(dp369
-VvisState
-p370
-V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}}
-p371
-sVdescription
-p372
-V
-sVtitle
-p373
-VSnort
-p374
-sVuiStateJSON
-p375
-V{}
-p376
-sVversion
-p377
-I1
-sVkibanaSavedObjectMeta
-p378
-(dp379
-VsearchSourceJSON
-p380
-V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p381
-sssV_index
-p382
-V.kibana
-p383
-sa(dp384
-V_score
-p385
-F1
-sV_type
-p386
-Vdashboard
-p387
-sV_id
-p388
-VMetron-Dashboard
-p389
-sV_source
-p390
-(dp391
-Vhits
-p392
-I0
-sVtimeRestore
-p393
-I00
-sVdescription
-p394
-V
-sVtitle
-p395
-VMetron Dashboard
-p396
-sVuiStateJSON
-p397
-V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}}
-p398
-sVpanelsJSON
-p399
-V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col"
:4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","i
p_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}]
-p400
-sVoptionsJSON
-p401
-V{"darkTheme":false}
-p402
-sVversion
-p403
-I1
-sVkibanaSavedObjectMeta
-p404
-(dp405
-VsearchSourceJSON
-p406
-V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]}
-p407
-sssV_index
-p408
-V.kibana
-p409
-sa(dp410
-V_score
-p411
-F1
-sV_type
-p412
-Vvisualization
-p413
-sV_id
-p414
-VSnort-Alert-Types
-p415
-sV_source
-p416
-(dp417
-VvisState
-p418
-V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}}
-p419
-sVdescription
-p420
-V
-sVtitle
-p421
-VSnort Alert Types
-p422
-sVuiStateJSON
-p423
-V{}
-p424
-sVversion
-p425
-I1
-sVkibanaSavedObjectMeta
-p426
-(dp427
-VsearchSourceJSON
-p428
-V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p429
-sssV_index
-p430
-V.kibana
-p431
-sa(dp432
-V_score
-p433
-F1
-sV_type
-p434
-Vvisualization
-p435
-sV_id
-p436
-VFrequent-DNS-Queries
-p437
-sV_source
-p438
-(dp439
-VvisState
-p440
-V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p441
-sVdescription
-p442
-V
-sVtitle
-p443
-VFrequent DNS Requests
-p444
-sVuiStateJSON
-p445
-V{}
-p446
-sVversion
-p447
-I1
-sVkibanaSavedObjectMeta
-p448
-(dp449
-VsearchSourceJSON
-p450
-V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p451
-sssV_index
-p452
-V.kibana
-p453
-sa(dp454
-V_score
-p455
-F1
-sV_type
-p456
-Vvisualization
-p457
-sV_id
-p458
-VLocation-Header
-p459
-sV_source
-p460
-(dp461
-VvisState
-p462
-V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}}
-p463
-sVdescription
-p464
-V
-sVtitle
-p465
-VEnrichment
-p466
-sVuiStateJSON
-p467
-V{}
-p468
-sVversion
-p469
-I1
-sVkibanaSavedObjectMeta
-p470
-(dp471
-VsearchSourceJSON
-p472
-V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p473
-sssV_index
-p474
-V.kibana
-p475
-sa(dp476
-V_score
-p477
-F1
-sV_type
-p478
-Vsearch
-p479
-sV_id
-p480
-Vweb-search
-p481
-sV_source
-p482
-(dp483
-Vsort
-p484
-(lp485
-Vtimestamp
-p486
-aVdesc
-p487
-asVhits
-p488
-I0
-sVdescription
-p489
-V
-sVtitle
-p490
-VWeb Requests
-p491
-sVversion
-p492
-I1
-sVkibanaSavedObjectMeta
-p493
-(dp494
-VsearchSourceJSON
-p495
-V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
-p496
-ssVcolumns
-p497
-(lp498
-Vmethod
-p499
-aVhost
-p500
-aVuri
-p501
-aVreferrer
-p502
-aVip_src_addr
-p503
-aVip_dst_addr
-p504
-assV_index
-p505
-V.kibana
-p506
-sa(dp507
-V_score
-p508
-F1
-sV_type
-p509
-Vindex-pattern
-p510
-sV_id
-p511
-Vsnort*
-p512
-sV_source
-p513
-(dp514
-Vfields
-p515
-V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyze
d":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scr
ipted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type"
:"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc
","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"n
umber","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"
is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","t
ype":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}]
-p516
-sVtimeFieldName
-p517
-Vtimestamp
-p518
-sVtitle
-p519
-Vsnort*
-p520
-ssV_index
-p521
-V.kibana
-p522
-sa(dp523
-V_score
-p524
-F1
-sV_type
-p525
-Vindex-pattern
-p526
-sV_id
-p527
-Vyaf*
-p528
-sV_source
-p529
-(dp530
-Vfields
-p531
-V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"strin
g","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"sc
ripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","
count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":
"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_value
s":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count"
:0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}]
-p532
-sVtimeFieldName
-p533
-Vtimestamp
-p534
-sVtitle
-p535
-Vyaf*
-p536
-ssV_index
-p537
-V.kibana
-p538
-sa(dp539
-V_score
-p540
-F1
-sV_type
-p541
-Vvisualization
-p542
-sV_id
-p543
-VDNS-Request(s)
-p544
-sV_source
-p545
-(dp546
-VvisState
-p547
-V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
-p548
-sVdescription
-p549
-V
-sVtitle
-p550
-VDNS Requests
-p551
-sVuiStateJSON
-p552
-V{}
-p553
-sVversion
-p554
-I1
-sVsavedSearchId
-p555
-Vdns-search
-p556
-sVkibanaSavedObjectMeta
-p557
-(dp558
-VsearchSourceJSON
-p559
-V{"filter":[]}
-p560
-sssV_index
-p561
-V.kibana
-p562
-sa(dp563
-V_score
-p564
-F1
-sV_type
-p565
-Vvisualization
-p566
-sV_id
-p567
-VHTTP(S)-Requests
-p568
-sV_source
-p569
-(dp570
-VvisState
-p571
-V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
-p572
-sVdescription
-p573
-V
-sVtitle
-p574
-VWeb Requests
-p575
-sVuiStateJSON
-p576
-V{}
-p577
-sVversion
-p578
-I1
-sVsavedSearchId
-p579
-Vweb-search
-p580
-sVkibanaSavedObjectMeta
-p581
-(dp582
-VsearchSourceJSON
-p583
-V{"filter":[]}
-p584
-sssV_index
-p585
-V.kibana
-p586
-sa(dp587
-V_score
-p588
-F1
-sV_type
-p589
-Vsearch
-p590
-sV_id
-p591
-Vdns-search
-p592
-sV_source
-p593
-(dp594
-Vsort
-p595
-(lp596
-Vtimestamp
-p597
-aVdesc
-p598
-asVhits
-p599
-I0
-sVdescription
-p600
-V
-sVtitle
-p601
-VDNS Requests
-p602
-sVversion
-p603
-I1
-sVkibanaSavedObjectMeta
-p604
-(dp605
-VsearchSourceJSON
-p606
-V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
-p607
-ssVcolumns
-p608
-(lp609
-Vquery
-p610
-aVqtype_name
-p611
-aVanswers
-p612
-aVip_src_addr
-p613
-aVip_dst_addr
-p614
-assV_index
-p615
-V.kibana
-p616
-sa(dp617
-V_score
-p618
-F1
-sV_type
-p619
-Vvisualization
-p620
-sV_id
-p621
-VFlow-Locations
-p622
-sV_source
-p623
-(dp624
-VvisState
-p625
-V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}}
-p626
-sVdescription
-p627
-V
-sVtitle
-p628
-VFlow Locations
-p629
-sVuiStateJSON
-p630
-V{}
-p631
-sVversion
-p632
-I1
-sVkibanaSavedObjectMeta
-p633
-(dp634
-VsearchSourceJSON
-p635
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p636
-sssV_index
-p637
-V.kibana
-p638
-sa(dp639
-V_score
-p640
-F1
-sV_type
-p641
-Vvisualization
-p642
-sV_id
-p643
-VUnusual-Referrers
-p644
-sV_source
-p645
-(dp646
-VvisState
-p647
-V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}}
-p648
-sVdescription
-p649
-V
-sVtitle
-p650
-VUnusual Referrers
-p651
-sVuiStateJSON
-p652
-V{}
-p653
-sVversion
-p654
-I1
-sVsavedSearchId
-p655
-Vweb-search
-p656
-sVkibanaSavedObjectMeta
-p657
-(dp658
-VsearchSourceJSON
-p659
-V{"filter":[]}
-p660
-sssV_index
-p661
-V.kibana
-p662
-sa(dp663
-V_score
-p664
-F1
-sV_type
-p665
-Vvisualization
-p666
-sV_id
-p667
-VFrequent-DNS-Requests
-p668
-sV_source
-p669
-(dp670
-VvisState
-p671
-V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}}
-p672
-sVdescription
-p673
-V
-sVtitle
-p674
-VFrequent DNS Requests
-p675
-sVuiStateJSON
-p676
-V{}
-p677
-sVversion
-p678
-I1
-sVkibanaSavedObjectMeta
-p679
-(dp680
-VsearchSourceJSON
-p681
-V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p682
-sssV_index
-p683
-V.kibana
-p684
-sa(dp685
-V_score
-p686
-F1
-sV_type
-p687
-Vvisualization
-p688
-sV_id
-p689
-VCountry
-p690
-sV_source
-p691
-(dp692
-VvisState
-p693
-V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
-p694
-sVdescription
-p695
-V
-sVtitle
-p696
-VBy Country
-p697
-sVuiStateJSON
-p698
-V{}
-p699
-sVversion
-p700
-I1
-sVkibanaSavedObjectMeta
-p701
-(dp702
-VsearchSourceJSON
-p703
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p704
-sssV_index
-p705
-V.kibana
-p706
-sa(dp707
-V_score
-p708
-F1
-sV_type
-p709
-Vvisualization
-p710
-sV_id
-p711
-VTop-Destinations
-p712
-sV_source
-p713
-(dp714
-VvisState
-p715
-V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}}
-p716
-sVdescription
-p717
-V
-sVtitle
-p718
-VTop Destinations
-p719
-sVuiStateJSON
-p720
-V{}
-p721
-sVversion
-p722
-I1
-sVkibanaSavedObjectMeta
-p723
-(dp724
-VsearchSourceJSON
-p725
-V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
-p726
-sssV_index
-p727
-V.kibana
-p728
-sa(dp729
-V_score
-p730
-F1
-sV_type
-p731
-Vvisualization
-p732
-sV_id
-p733
-VDNS-Requests-Header
-p734
-sV_source
-p735
-(dp736
-VvisState
-p737
-V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"}
-p738
-sVdescription
-p739
-V
-sVtitle
-p740
-VDNS Requests
-p741
-sVuiStateJSON
-p742
-V{}
-p743
-sVversion
-p744
-I1
-sVkibanaSavedObjectMeta
-p745
-(dp746
-VsearchSourceJSON
-p747
-V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
-p748
-sssV_index
-p749
-V.kibana
-p750
-sa(dp751
-V_score
-p752
-F1
-sV_type
-p753
-Vvisualization
-p754
-sV_id
-p755
-VYAF-Flows-Header
-p756
-sV_source
-p757
-(dp758
-VvisState
-p759
-V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}}
-p760
-sVdescription
-p761
-V
-sVtitle
-p762
-VYAF
-p763
-sVuiStateJSON
-p764
-V{}
-p765
-sVversion
-p766
-I1
-sVkibanaSavedObjectMeta
-p767
-(dp768
-VsearchSourceJSON
-p769
-V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
-p770
-sssV_index
-p771
-V.kibana
-p772
-sa.
\ No newline at end of file
[2/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
- def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
- """
- :arg host: hostname of the node (default: localhost)
- :arg port: port to use (integer, default: 9200)
- :arg url_prefix: optional url prefix for elasticsearch
- :arg timeout: default timeout in seconds (float, default: 10)
- """
- self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
-
- def get(self):
- """
- Get .kibana index from Elasticsearch
- """
- dotkibana = self.es.search(index='.kibana', size = 100)
- return dotkibana['hits']['hits']
-
- def load(self,filespec):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- data=[]
- with open(filespec,'rb') as fp:
- data = pickle.load(fp)
- return data
-
- def save(self,filename,data):
- """
- Save Index data on local filesystem
- :args filespec: path/filename for saved file
- """
- with open(filename,'wb') as fp:
- pickle.dump(data,fp)
-
- def put(self,data):
- """
- Bulk write data to Elasticsearch
- :args data: data to be written (note: index name is specified in data)
- """
- bulk(self.es,data)
-
- def main(self,args):
-
- if args.save:
- print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
- self.save(filename=args.filespec,data=di.get())
- else:
- """
- Loads Kibana Dashboard definition from disk and replaces .kibana on index
- :args filespec: path/filename for saved file
- """
- if not os.path.isfile(args.filespec):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
- self.es.indices.delete(index='.kibana', ignore=[400, 404])
- self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser()
- parser.add_argument("hostname", help="ES Hostname or IP", type=str)
- parser.add_argument("port", help="ES Port", type=int)
- parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
- parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
- args = parser.parse_args()
- di = DashboardIndex(host=args.hostname,port=args.port)
- di.main(args)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
deleted file mode 100755
index 7913c8c..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
+++ /dev/null
@@ -1,146 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-kibana_master
-
-"""
-
-
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions.format import format
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-
-import errno
-import os
-
-class Kibana(Script):
-
- def install(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Install Kibana Master")
-
- #TODO: Figure this out for all supported OSes
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[kibana-4.x]\n"
- "name=Kibana repository for 4.5.x packages\n"
- "baseurl=http://packages.elastic.co/kibana/4.5/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/kibana.repo")
-
- self.install_packages(env)
- self.loadtemplate(env)
-
- def configure(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Configure Kibana for Metron")
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- Directory(directories,
- # recursive=True,
- mode=0755,
- owner=params.kibana_user,
- group=params.kibana_user
- )
-
- File("{}/kibana.yml".format(params.conf_dir),
- owner=params.kibana_user,
- content=InlineTemplate(params.kibana_yml_template)
- )
-
- def stop(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Stop Kibana Master")
-
- Execute("service kibana stop")
-
- def start(self, env):
-
- import params
- env.set_params(params)
-
- self.configure(env)
-
- Logger.info("Start the Master")
-
- Execute("service kibana start")
-
- def restart(self,env):
-
- import params
- env.set_params(params)
-
- self.configure(env)
-
- Logger.info("Restarting the Master")
-
- Execute("service kibana restart")
-
- def status(self, env):
-
- import params
- env.set_params(params)
-
- Logger.info("Status of the Master")
-
- Execute("service kibana status")
-
- @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
- def loadtemplate(self,env):
-
- from dashboard.dashboardindex import DashboardIndex
-
- import params
- env.set_params(params)
-
- hostname = format("{es_host}")
- port = int(format("{es_port}"))
-
- Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname,port))
- di = DashboardIndex(host=hostname,port=port)
-
- #Loads Kibana Dashboard definition from disk and replaces .kibana on index
- templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)),'dashboard','dashboard.p')
- if not os.path.isfile(templateFile):
- raise IOError(
- errno.ENOENT, os.strerror(errno.ENOENT), templateFile)
-
- Logger.info("Deleting .kibana index from Elasticsearch")
-
- di.es.indices.delete(index='.kibana', ignore=[400, 404])
-
- Logger.info("Loading .kibana index from %s" % templateFile)
-
- di.put(data=di.load(filespec=templateFile))
-
-
-if __name__ == "__main__":
- Kibana().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
deleted file mode 100755
index 03fae0b..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Kibana Params configurations
-
-"""
-
-
-from resource_management.libraries.script import Script
-from resource_management.libraries.functions import format
-from urlparse import urlparse
-
-# server configurations
-config = Script.get_config()
-
-kibana_home = '/usr/share/kibana/'
-kibana_bin = '/usr/share/kibana/bin/'
-
-conf_dir = "/opt/kibana/config/"
-kibana_user = config['configurations']['kibana-env']['kibana_user']
-user_group = config['configurations']['kibana-env']['user_group']
-log_dir = config['configurations']['kibana-env']['kibana_log_dir']
-pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
-pid_file = format("{pid_dir}/kibanasearch.pid")
-es_url = config['configurations']['kibana-env']['kibana_es_url']
-parsed = urlparse(es_url)
-es_host = parsed.netloc.split(':')[0]
-es_port = parsed.netloc.split(':')[1]
-kibana_port = config['configurations']['kibana-env']['kibana_server_port']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-kibana_yml_template = config['configurations']['kibana-site']['content']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
deleted file mode 100755
index 448e102..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "metron_ui",
- "label": "Metron UI",
- "requires_user_name": "false",
- "component_name": "KIBANA_MASTER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "kibana_server_port",
- "http_default_port": "5601",
- "https_property": "kibana_server_port",
- "https_default_port": "5601",
- "regex": "^(\\d+)$",
- "site": "kibana-env"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
deleted file mode 100755
index 71966ea..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.0BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron parsers</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_zookeeper_config_dir</name>
- <value>config/zookeeper</value>
- <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
- <display-name>Metron Zookeeper config dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <description>The group for Metron</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
deleted file mode 100755
index f1e5ab7..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,108 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>PARSERS</name>
- <displayName>Metron Parsers</displayName>
- <comment>Parsers</comment>
- <version>0.2.0BETA</version>
-
- <components>
- <component>
- <name>PARSER_MASTER</name>
- <displayName>Metron Parser Topologies</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/parser_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <configuration-dependencies>
- <config-type>metron-parsers</config-type>
- </configuration-dependencies>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-parsers</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-parsers</config-type>
- </configuration-dependencies>
-
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
-
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
deleted file mode 100755
index 6b33e55..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import subprocess
-import time
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import Template
-from resource_management.libraries.functions import format
-
-
-# Wrap major operations and functionality in this class
-class Commands:
- __params = None
- __parser_list = None
- __configured = False
-
- def __init__(self, params):
- if params is None:
- raise ValueError("params argument is required for initialization")
- self.__params = params
- self.__parser_list = self.__get_parsers(params)
- self.__configured = os.path.isfile(self.__params.configured_flag_file)
-
- # get list of parsers
- def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
-
- def is_configured(self):
- return self.__configured
-
- def set_configured(self):
- File(self.__params.configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0775)
-
- # Possible storm topology status states
- # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html
- class StormStatus:
- ACTIVE = "ACTIVE"
- INACTIVE = "INACTIVE"
- KILLED = "KILLED"
- REBALANCING = "REBALANCING"
-
- def init_parsers(self):
- Logger.info(
- "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir,
- self.__params.metron_apps_dir))
- self.__params.HdfsResource(self.__params.metron_apps_dir,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- mode=0775,
- source=self.__params.local_grok_patterns_dir)
-
- Logger.info("Creating global.json file")
- File(self.__params.metron_zookeeper_config_path + '/global.json',
- content=Template("metron-global.json"),
- owner=self.__params.metron_user,
- mode=0775)
- Logger.info("Done initializing parser configuration")
-
- def get_parser_list(self):
- return self.__parser_list
-
- def setup_repo(self):
- def local_repo():
- Logger.info("Setting up local repo")
- Execute("yum -y install createrepo")
- Execute("createrepo /localrepo")
- Execute("chmod -R o-w+r /localrepo")
- Execute("echo \"[METRON-0.2.0BETA]\n"
- "name=Metron 0.2.0BETA packages\n"
- "baseurl=file:///localrepo\n"
- "gpgcheck=0\n"
- "enabled=1\" > /etc/yum.repos.d/local.repo")
-
- def remote_repo():
- print('Using remote repo')
-
- yum_repo_types = {
- 'local': local_repo,
- 'remote': remote_repo
- }
- repo_type = self.__params.yum_repo_type
- if repo_type in yum_repo_types:
- yum_repo_types[repo_type]()
- else:
- raise ValueError("Unsupported repo type '{}'".format(repo_type))
-
- def init_kafka_topics(self):
- Logger.info('Creating Kafka topics')
- command_template = """{}/kafka-topics.sh \
- --zookeeper {} \
- --create \
- --topic {} \
- --partitions {} \
- --replication-factor {} \
- --config retention.bytes={}"""
- num_partitions = 1
- replication_factor = 1
- retention_gigabytes = 10
- retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
- Logger.info("Creating main topics for parsers")
- for parser_name in self.get_parser_list():
- Logger.info("Creating topic'{}'".format(parser_name))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- parser_name,
- num_partitions,
- replication_factor,
- retention_bytes))
- Logger.info("Creating topics for error handling")
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_invalid",
- num_partitions,
- replication_factor,
- retention_bytes))
- Execute(command_template.format(self.__params.kafka_bin_dir,
- self.__params.zookeeper_quorum,
- "parser_error",
- num_partitions, replication_factor,
- retention_bytes))
- Logger.info("Done creating Kafka topics")
-
- def init_parser_config(self):
- Logger.info('Loading parser config into ZooKeeper')
- Execute(format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
- path=format("{java_home}/bin")
- )
-
- def start_parser_topologies(self):
- Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list()))
- start_cmd_template = """{}/bin/start_parser_topology.sh \
- -k {} \
- -z {} \
- -s {}"""
- for parser in self.get_parser_list():
- Logger.info('Starting ' + parser)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
- self.__params.zookeeper_quorum, parser))
-
- Logger.info('Finished starting parser topologies')
-
- def stop_parser_topologies(self):
- Logger.info('Stopping parsers')
- for parser in self.get_parser_list():
- Logger.info('Stopping ' + parser)
- stop_cmd = 'storm kill ' + parser
- Execute(stop_cmd)
- Logger.info('Done stopping parser topologies')
-
- def restart_parser_topologies(self):
- Logger.info('Restarting the parser topologies')
- self.stop_parser_topologies()
- attempt_count = 0
- while self.topologies_exist():
- if attempt_count > 2:
- raise Exception("Unable to kill topologies")
- attempt_count += 1
- time.sleep(10)
- self.start_parser_topologies()
- Logger.info('Done restarting the parser topologies')
-
- def topologies_exist(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- if stdout_lines:
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- return True
- return False
-
- def topologies_running(self):
- cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = cmd_open.communicate()
- stdout_lines = stdout.splitlines()
- all_running = False
- if stdout_lines:
- all_running = True
- status_lines = self.__get_status_lines(stdout_lines)
- for parser in self.get_parser_list():
- parser_found = False
- is_running = False
- for line in status_lines:
- items = re.sub('[\s]+', ' ', line).split()
- if items and items[0] == parser:
- status = items[1]
- parser_found = True
- is_running = self.__is_running(status)
- all_running &= parser_found and is_running
- return all_running
-
- def __get_status_lines(self, lines):
- status_lines = []
- do_stat = False
- skipped = 0
- for line in lines:
- if line.startswith("Topology_name"):
- do_stat = True
- if do_stat and skipped == 2:
- status_lines += [line]
- elif do_stat:
- skipped += 1
- return status_lines
-
- def __is_running(self, status):
- return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING]
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
deleted file mode 100755
index 242460e..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
deleted file mode 100755
index 953435d..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.expect import expect
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
deleted file mode 100755
index de7dfc9..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import functools
-import os
-
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-hostname = config['hostname']
-#print(config['configurations'])
-metron_home = config['configurations']['metron-parsers']['metron_home']
-parsers = config['configurations']['metron-parsers']['parsers']
-metron_user = config['configurations']['metron-parsers']['metron_user']
-metron_group = config['configurations']['metron-parsers']['metron_group']
-metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir']
-#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir
-metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
-configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured'
-yum_repo_type = 'local'
-
-# hadoop params
-stack_root = Script.get_stack_root()
-hadoop_home_dir = stack_select.get_hadoop_dir("home")
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-kafka_home = os.path.join(stack_root, "current", "kafka-broker")
-kafka_bin_dir = os.path.join(kafka_home, "bin")
-
-# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
-
-# Kafka
-kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
-has_kafka_host = not len(kafka_hosts) == 0
-kafka_brokers = None
-if has_kafka_host:
- if 'port' in config['configurations']['kafka-broker']:
- kafka_broker_port = config['configurations']['kafka-broker']['port']
- else:
- kafka_broker_port = '6667'
- kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
- kafka_brokers += ':' + kafka_broker_port
-
-metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir']
-
-local_grok_patterns_dir = format("{metron_home}/patterns")
-hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
-
-# for create_hdfs_directory
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hostname = config["hostname"]
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-
-# create partial functions with common arguments for every HdfsResource call
-# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
deleted file mode 100755
index 4d11b35..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
deleted file mode 100755
index ee83414..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-parsers = "bro,yaf,snort"
-configured_flag_file = ""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
deleted file mode 100755
index 4a5b910..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class ParserMaster(Script):
- def get_component_name(self):
- # TODO add this at some point - currently will cause problems with hdp-select
- # return "parser-master"
- pass
-
- def install(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.setup_repo()
- Logger.info('Install RPM packages')
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- from params import params
- env.set_params(params)
-
- def start(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- if not commands.is_configured():
- commands.init_parsers()
- commands.init_kafka_topics()
- commands.init_parser_config()
- commands.set_configured()
- commands.start_parser_topologies()
-
- def stop(self, env, upgrade_type=None):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.stop_parser_topologies()
-
- def status(self, env):
- from params import status_params
- env.set_params(status_params)
- commands = Commands(status_params)
- if not commands.topologies_running():
- raise ComponentIsNotRunning()
-
- def restart(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- commands.restart_parser_topologies()
-
-
-if __name__ == "__main__":
- ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
deleted file mode 100755
index 6ef7e4a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-from resource_management.libraries.script import Script
-
-from commands import Commands
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- from params import params
- env.set_params(params)
- commands = Commands(params)
- all_found = commands.topologies_running()
- if all_found:
- exit(0)
- else:
- exit(1)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
deleted file mode 100755
index 0967ef4..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
+++ /dev/null
@@ -1 +0,0 @@
-{}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
deleted file mode 100755
index ee1b225..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol":
- {
- "type":"HTTP_ONLY"
- },
-
- "links": [
- {
- "name": "storm_ui",
- "label": "Storm UI",
- "requires_user_name": "false",
- "component_name": "STORM_UI_SERVER",
- "url":"%@://%@:%@/",
- "port":{
- "http_property": "ui.port",
- "http_default_port": "8744",
- "https_property": "ui.port",
- "https_default_port": "8744",
- "regex": "^(\\d+)$",
- "site": "storm-site"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
deleted file mode 100755
index 84b466a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import imp
-import os
-import traceback
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
-PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
-
-try:
- with open(PARENT_FILE, 'rb') as fp:
- service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
-except Exception as e:
- traceback.print_exc()
- print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE))
-
-
-class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor):
- # colocate Metron Parser Master with KAFKA_BROKERs
- def TODO_colocateService(self, hostsComponentsMap, serviceComponents):
- parsersMasterComponent = [component for component in serviceComponents if
- component["StackServiceComponents"]["component_name"] == "PARSER_MASTER"][0]
- if not self.isComponentHostsPopulated(parsersMasterComponent):
- for hostName in hostsComponentsMap.keys():
- hostComponents = hostsComponentsMap[hostName]
- if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"} not in hostComponents:
- hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"})
- if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"} in hostComponents:
- hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"})
-
- def TODO_getServiceComponentLayoutValidations(self, services, hosts):
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
-
- masterHosts = self.getHosts(componentsList, "PARSER_MASTER")
- expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER"))
-
- items = []
-
- mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts)))
- if len(mismatchHosts) > 0:
- hostsString = ', '.join(mismatchHosts)
- message = "Metron Parsers Master must be installed on Kafka Brokers. " \
- "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(
- len(mismatchHosts), hostsString)
- items.append(
- {"type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PARSER_MASTER'})
-
- return items
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
deleted file mode 100755
index e0292cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>PARSERS</name>
- <version>0.2.0BETA</version>
- <extends>common-services/PARSERS/0.2.0BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
deleted file mode 100755
index c861646..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "_comment": "Record format:",
- "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps": {
- "_comment": "dependencies for Metron Parsers",
- "PARSER_MASTER-START": [
- "NAMENODE-START",
- "ZOOKEEPER_SERVER-START",
- "KAFKA_BROKER-START",
- "STORM_REST_API-START"
- ],
- "_comment": "<SERVICE_NAME>_SERVICE_CHECK-SERVICE_CHECK",
- "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [
- "PARSER_MASTER-START"
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
deleted file mode 100755
index 361f5f5..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/ELASTICSEARCH/metainfo.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <displayName>Elasticsearch</displayName>
- <comment>Indexing and Search</comment>
- <version>2.3.3</version>
- <extends>common-services/ELASTICSEARCH/2.3.3</extends>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>elasticsearch-2.3.3</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- </service>
- </services>
-</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
deleted file mode 100755
index e406780..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>INDEXING</name>
- <version>0.2.0BETA</version>
- <extends>common-services/INDEXING/0.2.0BETA</extends>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
deleted file mode 100755
index f58c59c..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
- "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
deleted file mode 100755
index 494f71b..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/KIBANA/metainfo.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>KIBANA</name>
- <extends>common-services/KIBANA/4.5.1</extends>
- </service>
- </services>
-</metainfo>
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 7f78806..6b8f195 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -220,6 +220,7 @@ This package installs the Metron Enrichment files
%dir %{metron_home}/config
%dir %{metron_home}/config/zookeeper
%dir %{metron_home}/config/zookeeper/enrichments
+%dir %{metron_home}/ddl
%dir %{metron_home}/flux
%dir %{metron_home}/flux/enrichment
%{metron_home}/bin/latency_summarizer.sh
@@ -229,6 +230,7 @@ This package installs the Metron Enrichment files
%{metron_home}/config/zookeeper/enrichments/snort.json
%{metron_home}/config/zookeeper/enrichments/websphere.json
%{metron_home}/config/zookeeper/enrichments/yaf.json
+%{metron_home}/ddl/geoip_ddl.sql
%{metron_home}/flux/enrichment/remote.yaml
%exclude %{metron_home}/flux/enrichment/test.yaml
%attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
new file mode 100644
index 0000000..92414c2
--- /dev/null
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -0,0 +1,218 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-rpm</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-rpm</name>
+ <version>0.2.0BETA</version>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-deployment</artifactId>
+ <version>0.2.0BETA</version>
+ <relativePath>../../..</relativePath>
+ </parent>
+ <description>RPM Builder for Apache Metron</description>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <rpm.docker.tag>rpm-docker</rpm.docker.tag>
+ <metron_dir>${parent.relativePath}/..</metron_dir>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.1</version>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <echo>******** Displaying value of property ********</echo>
+ <echo>${metron_dir}</echo>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>BUILD</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>BUILDROOT</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>RPMS</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>RPMS/noarch</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>SOURCES</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ <fileset>
+ <directory>SRPMS</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.5.0</version>
+ <executions>
+ <execution>
+ <id>docker-build</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>docker</executable>
+ <arguments>
+ <argument>build</argument>
+ <argument>-f</argument>
+ <argument>Dockerfile</argument>
+ <argument>-t</argument>
+ <argument>${rpm.docker.tag}</argument>
+ <argument>.</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>rpm-build</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>docker</executable>
+ <arguments>
+ <argument>run</argument>
+ <argument>-v</argument>
+ <argument>${project.basedir}:/root</argument>
+ <argument>${rpm.docker.tag}:latest</argument>
+ <argument>/bin/bash</argument>
+ <argument>-c</argument>
+ <argument>./build.sh ${project.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>copy-rpm-sources</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/SOURCES</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-common/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-data-management/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-elasticsearch/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-enrichment/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-indexing/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-parsers/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-pcap-backend/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${metron_dir}/metron-platform/metron-solr/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-rpms-target</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/target/RPMS</outputDirectory>
+ <resources>
+ <resource>
+ <directory>RPMS</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index 24b8498..d4b05f1 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -29,166 +29,21 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<rpm.docker.tag>rpm-docker</rpm.docker.tag>
</properties>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>3.0.0</version>
- <configuration>
- <filesets>
- <fileset>
- <directory>packaging/docker/rpm-docker/BUILD</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/BUILDROOT</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/RPMS</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/RPMS/noarch</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/SOURCES</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- <fileset>
- <directory>packaging/docker/rpm-docker/SRPMS</directory>
- <followSymlinks>false</followSymlinks>
- </fileset>
- </filesets>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
- <executions>
- <execution>
- <id>docker-build</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>docker</executable>
- <arguments>
- <argument>build</argument>
- <argument>-f</argument>
- <argument>packaging/docker/rpm-docker/Dockerfile</argument>
- <argument>-t</argument>
- <argument>${rpm.docker.tag}</argument>
- <argument>.</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>rpm-build</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>docker</executable>
- <arguments>
- <argument>run</argument>
- <argument>-v</argument>
- <argument>${project.basedir}/packaging/docker/rpm-docker:/root</argument>
- <argument>${rpm.docker.tag}:latest</argument>
- <argument>/bin/bash</argument>
- <argument>-c</argument>
- <argument>./build.sh ${project.version}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>3.0.1</version>
- <executions>
- <execution>
- <id>copy-rpm-sources</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.basedir}/packaging/docker/rpm-docker/SOURCES</outputDirectory>
- <resources>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-common/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-data-management/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-elasticsearch/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-enrichment/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-indexing/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-parsers/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-pcap-backend/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- <resource>
- <directory>${project.basedir}/../metron-platform/metron-solr/target/</directory>
- <includes>
- <include>*.tar.gz</include>
- </includes>
- </resource>
- </resources>
- </configuration>
- </execution>
- <execution>
- <id>copy-rpms-target</id>
- <phase>package</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.basedir}/target/RPMS</outputDirectory>
- <resources>
- <resource>
- <directory>packaging/docker/rpm-docker/RPMS</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+ <profiles>
+ <profile>
+ <id>build-rpms</id>
+ <modules>
+ <module>packaging/docker/rpm-docker</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>default</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <modules>
+ <module>packaging/ambari/metron-mpack</module>
+ </modules>
+ </profile>
+ </profiles>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml
index 8612cfa..eda4a06 100644
--- a/metron-deployment/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/roles/ambari_common/defaults/main.yml
@@ -18,4 +18,4 @@
hadoop_logrotate_frequency: daily
hadoop_logrotate_retention: 30
-rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.2.2.0/ambari.repo"
\ No newline at end of file
+rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.0.1/ambari.repo"
\ No newline at end of file
[4/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
new file mode 100755
index 0000000..b3fb809
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_commands.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class ParserCommands:
+ __params = None
+ __parser_list = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__parser_list = self.__get_parsers(params)
+ self.__configured = os.path.isfile(self.__params.parsers_configured_flag_file)
+
+ # get list of parsers
+ def __get_parsers(self, params):
+ return params.parsers.replace(' ', '').split(',')
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.parsers_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def init_parsers(self):
+ Logger.info(
+ "Copying grok patterns from local directory '{0}' to HDFS '{1}'".format(self.__params.local_grok_patterns_dir,
+ self.__params.metron_apps_dir))
+ self.__params.HdfsResource(self.__params.metron_apps_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ mode=0775,
+ source=self.__params.local_grok_patterns_dir)
+
+ Logger.info("Done initializing parser configuration")
+
+ def get_parser_list(self):
+ return self.__parser_list
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating main topics for parsers")
+ for parser_name in self.get_parser_list():
+ Logger.info("Creating topic'{0}'".format(parser_name))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ parser_name,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Creating topics for error handling")
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_invalid",
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ "parser_error",
+ num_partitions, replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_parser_topologies(self):
+ Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ start_cmd_template = """{0}/bin/start_parser_topology.sh \
+ -k {1} \
+ -z {2} \
+ -s {3}"""
+ for parser in self.get_parser_list():
+ Logger.info('Starting ' + parser)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+ self.__params.zookeeper_quorum, parser))
+
+ Logger.info('Finished starting parser topologies')
+
+ def stop_parser_topologies(self):
+ Logger.info('Stopping parsers')
+ for parser in self.get_parser_list():
+ Logger.info('Stopping ' + parser)
+ stop_cmd = 'storm kill ' + parser
+ Execute(stop_cmd)
+ Logger.info('Done stopping parser topologies')
+
+ def restart_parser_topologies(self, env):
+ Logger.info('Restarting the parser topologies')
+ self.stop_parser_topologies()
+ attempt_count = 0
+ while self.topologies_running(env):
+ if attempt_count > 2:
+ raise Exception("Unable to kill topologies")
+ attempt_count += 1
+ time.sleep(10)
+ self.start_parser_topologies()
+ Logger.info('Done restarting the parser topologies')
+
+ def topologies_exist(self):
+ cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = cmd_open.communicate()
+ stdout_lines = stdout.splitlines()
+ if stdout_lines:
+ status_lines = self.__get_status_lines(stdout_lines)
+ for parser in self.get_parser_list():
+ for line in status_lines:
+ items = re.sub('[\s]+', ' ', line).split()
+ if items and items[0] == parser:
+ return True
+ return False
+
+ def topologies_running(self, env):
+ env.set_params(self.__params)
+ all_running = True
+ topologies = metron_service.get_running_topologies()
+ for parser in self.get_parser_list():
+ parser_found = False
+ is_running = False
+ if parser in topologies:
+ parser_found = True
+ is_running = topologies[parser] in ['ACTIVE', 'REBALANCING']
+ all_running &= parser_found and is_running
+ return all_running
+
+ def __get_status_lines(self, lines):
+ status_lines = []
+ do_stat = False
+ skipped = 0
+ for line in lines:
+ if line.startswith("Topology_name"):
+ do_stat = True
+ if do_stat and skipped == 2:
+ status_lines += [line]
+ elif do_stat:
+ skipped += 1
+ return status_lines
+
+ def __is_running(self, status):
+ return status in ['ACTIVE', 'REBALANCING']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..3758873
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/parser_master.py
@@ -0,0 +1,89 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+import metron_service
+from parser_commands import ParserCommands
+
+
+class ParserMaster(Script):
+ def get_component_name(self):
+ # TODO add this at some point - currently will cause problems with hdp-select
+ # return "parser-master"
+ pass
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ metron_service.load_global_config(params)
+ commands = ParserCommands(params)
+ if not commands.is_configured():
+ commands.init_parsers()
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.start_parser_topologies()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = ParserCommands(params)
+ commands.stop_parser_topologies()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = ParserCommands(status_params)
+ if not commands.topologies_running(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = ParserCommands(params)
+ commands.restart_parser_topologies(env)
+
+ def servicechecktest(self, env):
+ from params import params
+ env.set_params(params)
+ from service_check import ServiceCheck
+ service_check = ServiceCheck()
+ Logger.info('Service Check Test')
+ service_check.service_check(env)
+
+
+if __name__ == "__main__":
+ ParserMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..7dd9dfb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from indexing_commands import IndexingCommands
+from parser_commands import ParserCommands
+
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ from params import params
+ parsercommands = ParserCommands(params)
+ indexingcommands = IndexingCommands(params)
+ all_found = parsercommands.topologies_running(env) and indexingcommands.is_topology_active(env)
+ if all_found:
+ exit(0)
+ else:
+ exit(1)
+
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
new file mode 100755
index 0000000..bab2f52
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/templates/enrichment.properties.j2
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip={{mysql_host}}
+mysql.port={{mysql_port}}
+mysql.username={{enrichment_metron_user}}
+mysql.password={{enrichment_metron_user_passwd}}
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table={{enrichment_table}}
+enrichment.simple.hbase.cf={{enrichment_cf}}
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table={{threatintel_table}}
+threat.intel.tracker.cf={{threatintel_cf}}
+threat.intel.simple.hbase.table={{threatintel_table}}
+threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.ip.table=
+threat.intel.ip.cf=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "storm_ui",
+ "label": "Storm UI",
+ "requires_user_name": "false",
+ "component_name": "STORM_UI_SERVER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "ui.port",
+ "http_default_port": "8744",
+ "https_property": "ui.port",
+ "https_default_port": "8744",
+ "regex": "^(\\d+)$",
+ "site": "storm-site"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..99dec9b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -0,0 +1,85 @@
+{
+ "type": "full-release",
+ "name": "metron-ambari.mpack",
+ "version": "1.0.0.0",
+ "description": "Ambari Management Pack for Apache Metron",
+ "prerequisites": {
+ "min-ambari-version": "2.4.0.0",
+ "min-stack-versions": [
+ {
+ "stack_name": "HDP",
+ "stack_version": "2.3.0"
+ }
+ ]
+ },
+ "artifacts": [
+ {
+ "name": "METRON-common-services",
+ "type" : "service-definitions",
+ "source_dir" : "common-services"
+ },
+ {
+ "name" : "METRON-addon-services",
+ "type" : "stack-addon-service-definitions",
+ "source_dir": "addon-services",
+ "service_versions_map": [
+ {
+ "service_name" : "KIBANA",
+ "service_version" : "4.5.1",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+ ]
+ },
+ {
+ "service_name" : "ELASTICSEARCH",
+ "service_version" : "2.3.3",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+
+ ]
+ },
+ {
+ "service_name" : "METRON",
+ "service_version" : "0.2.0BETA",
+ "applicable_stacks" : [
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.3"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.4"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.5"
+ }
+
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
deleted file mode 100755
index 2939c28..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_user</name>
- <value>elasticsearch</value>
- <property-type>USER</property-type>
- <description>The user for Elasticsearch</description>
- </property>
- <property>
- <name>user_group</name>
- <value>elasticsearch</value>
- <description>The group for Elasticsearch</description>
- </property>
- <property>
- <name>elastic_log_dir</name>
- <value>/var/log/elasticsearch</value>
- <description>Log directory for elastic</description>
- </property>
- <property>
- <name>elastic_pid_dir</name>
- <value>/var/run/elasticsearch</value>
- <description>The directory for pid files</description>
- </property>
-
- <!-- elasticsearch-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-#!/bin/bash
-
-# Set ELASTICSEARCH specific environment variables here.
-
-# The java implementation to use.
-export JAVA_HOME={{java64_home}}
-export PATH=$PATH:$JAVA_HOME/bin
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
deleted file mode 100755
index fb3a443..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
+++ /dev/null
@@ -1,180 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Elastic search Configurations -->
-
-<configuration supports_final="true">
- <!-- Configurations -->
- <property>
- <name>cluster_name</name>
- <value>metron</value>
- <description>Cluster name identifies your cluster</description>
- </property>
- <property>
- <name>zen_discovery_ping_unicast_hosts</name>
- <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.-->
- <value></value>
- <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description>
- </property>
- <property>
- <name>index_number_of_shards</name>
- <value>4</value>
- <description>Set the number of shards (splits) of an index</description>
- </property>
- <property>
- <name>index_number_of_replicas</name>
- <value>2</value>
- <description>Set the number of replicas (additional copies) of an index</description>
- </property>
- <!-- Logging Configurations -->
- <property>
- <name>path_data</name>
- <value>"/opt/lmm/es_data"</value>
- <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description>
- </property>
- <!-- Discovery -->
- <property>
- <name>transport_tcp_port</name>
- <value>9300-9400</value>
- <description>Set a custom port for the node to node communication</description>
- </property>
- <property>
- <name>http_port</name>
- <value>9200-9300</value>
- <description>Set a custom port to listen for HTTP traffic</description>
- </property>
- <!-- Discovery -->
- <property>
- <name>discovery_zen_ping_multicast_enabled</name>
- <value>false</value>
- <description>master eligible nodes</description>
- </property>
- <property>
- <name>discovery_zen_ping_timeout</name>
- <value>3s</value>
- <description>Wait for ping responses for master discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_interval</name>
- <value>15s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_timeout</name>
- <value>60s</value>
- <description>Wait for ping for cluster discovery</description>
- </property>
- <property>
- <name>discovery_zen_fd_ping_retries</name>
- <value>5</value>
- <description>Number of ping retries before blacklisting</description>
- </property>
- <!-- Gateway -->
- <property>
- <name>gateway_recover_after_data_nodes</name>
- <value>3</value>
- <description>Recover as long as this many data or master nodes have joined the cluster.</description>
- </property>
- <property>
- <name>recover_after_time</name>
- <value>15m</value>
- <description>recover_after_time</description>
- </property>
- <property>
- <name>expected_data_nodes</name>
- <value>0</value>
- <description>expected_data_nodes</description>
- </property>
- <!-- Index -->
- <property>
- <name>index_merge_scheduler_max_thread_count</name>
- <value>5</value>
- <description>index.merge.scheduler.max_thread_count</description>
- </property>
- <property>
- <name>indices_memory_index_store_throttle_type</name>
- <value>none</value>
- <description>index_store_throttle_type</description>
- </property>
- <property>
- <name>index_refresh_interval</name>
- <value>1s</value>
- <description>index refresh interval</description>
- </property>
- <property>
- <name>index_translog_flush_threshold_size</name>
- <value>5g</value>
- <description>index_translog_flush_threshold_size</description>
- </property>
- <property>
- <name>indices_memory_index_buffer_size</name>
- <value>10%</value>
- <description>Percentage of heap used for write buffers</description>
- </property>
- <property>
- <name>bootstrap_mlockall</name>
- <value>true</value>
- <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description>
- </property>
- <property>
- <name>threadpool_bulk_queue_size</name>
- <value>3000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description>
- </property>
- <property>
- <name>threadpool_index_queue_size</name>
- <value>1000</value>
- <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description>
- </property>
- <property>
- <name>indices_cluster_send_refresh_mapping</name>
- <value>false</value>
- <description>In order to make the index request more efficient, we have set this property on our data nodes</description>
- </property>
- <property>
- <name>indices_fielddata_cache_size</name>
- <value>25%</value>
- <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_high</name>
- <value>0.99</value>
- <description>Property used when multiple drives are used to understand max thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_threshold_enabled</name>
- <value>true</value>
- <description>Property used when multiple drives are used to understand if thresholding is active</description>
- </property>
- <property>
- <name>cluster_routing_allocation_disk_watermark_low</name>
- <value>.97</value>
- <description>Property used when multiple drives are used to understand min thresholds</description>
- </property>
- <property>
- <name>cluster_routing_allocation_node_concurrent_recoveries</name>
- <value>4</value>
- <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description>
- </property>
- <property>
- <name>network_host</name>
- <value>_lo_,_eth0_</value>
- <description>Network interface(s) will bind to. </description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
deleted file mode 100755
index 58e4916..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
- <property>
- <name>elastic_home</name>
- <value>/usr/share/elasticsearch/</value>
- <description>Elasticsearch Home Directory</description>
- </property>
- <property>
- <name>data_dir</name>
- <value>/var/lib/elasticsearch/</value>
- <description>Elasticsearch Data Directory</description>
- </property>
- <property>
- <name>work_dir</name>
- <value>/tmp/elasticsearch/</value>
- <description>Elasticsearch Work Directory</description>
- </property>
- <property>
- <name>conf_dir</name>
- <value>/etc/elasticsearch/</value>
- <description>Elasticsearch Configuration Directory</description>
- </property>
- <property>
- <name>heap_size</name>
- <value>128m</value>
- <description>Heap size</description>
- </property>
- <property>
- <name>max_open_files</name>
- <value>65535</value>
- <description>Maximum number of open files</description>
- </property>
- <property>
- <name>max_map_count</name>
- <value>262144</value>
- <description>Maximum number of memory map areas for process</description>
- </property>
-
- <!--/etc/sysconfig/elasticsearch-->
- <property>
- <name>content</name>
- <description>This is the jinja template for elastic-env.sh file</description>
- <value>
-# Directory where the Elasticsearch binary distribution resides
-ES_HOME={{elastic_home}}
-
-# Heap Size (defaults to 256m min, 1g max)
-ES_HEAP_SIZE={{heap_size}}
-
-# Maximum number of open files
-MAX_OPEN_FILES={{max_open_files}}
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-MAX_MAP_COUNT={{max_map_count}}
-
-# Elasticsearch log directory
-LOG_DIR={{log_dir}}
-
-# Elasticsearch data directory
-DATA_DIR={{data_dir}}
-
-# Elasticsearch work directory
-WORK_DIR={{work_dir}}
-
-# Elasticsearch conf directory
-CONF_DIR={{conf_dir}}
-
-# User to run as, change this to a specific elasticsearch user if possible
-# Also make sure, this user can write into the log directories in case you change them
-# This setting only works for the init script, but has to be configured separately for systemd startup
-ES_USER={{elastic_user}}
-
-# Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
--XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
--XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8"
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
deleted file mode 100755
index a420131..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
+++ /dev/null
@@ -1,73 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ELASTICSEARCH</name>
- <displayName>Elasticsearch</displayName>
- <comment>Indexing and Search</comment>
- <version>2.3.3</version>
- <components>
- <component>
- <name>ES_MASTER</name>
- <displayName>Elasticsearch Master-Eligible Node</displayName>
- <category>MASTER</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/elastic_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- <component>
- <name>ES_SLAVE</name>
- <displayName>Elasticsearch Data Node</displayName>
- <category>SLAVE</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/elastic_slave.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>elasticsearch-2.3.3</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
- <configuration-dependencies>
- <config-type>elastic-env</config-type>
- <config-type>elastic-site</config-type>
- <config-type>elastic-sysconfig</config-type>
- </configuration-dependencies>
- <restartRequiredAfterChange>true</restartRequiredAfterChange>
- </service>
- </services>
-</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
deleted file mode 100755
index bd858cd..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-
-
-def elastic():
- print "INSIDE THE %s" % __file__
- import params
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
-
- Directory(directories,
- create_parents=True,
- # recursive=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master env: ""{}/elastic-env.sh".format(params.conf_dir)
- File("{}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- configurations = params.config['configurations']['elastic-site']
-
- print "Master yml: ""{}/elasticsearch.yml".format(params.conf_dir)
- File("{}/elasticsearch.yml".format(params.conf_dir),
- content=Template(
- "elasticsearch.master.yaml.j2",
- configurations=configurations),
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master sysconfig: /etc/sysconfig/elasticsearch"
- File(format("/etc/sysconfig/elasticsearch"),
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
deleted file mode 100755
index 5fc29cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from elastic import elastic
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.script import Script
-
-
-class Elasticsearch(Script):
- def install(self, env):
- import params
- env.set_params(params)
-
- print 'Install the Master'
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[elasticsearch-2.x]\n"
- "name=Elasticsearch repository for 2.x packages\n"
- "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
-
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
-
- elastic()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
- print 'Stop the Master'
- Execute(stop_cmd)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
-
- self.configure(env)
- start_cmd = format("service elasticsearch start")
- print 'Start the Master'
- Execute(start_cmd)
-
- def status(self, env):
- import params
- env.set_params(params)
- status_cmd = format("service elasticsearch status")
- print 'Status of the Master'
- Execute(status_cmd)
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- restart_cmd = format("service elasticsearch restart")
- print 'Restarting the Master'
- Execute(restart_cmd)
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
deleted file mode 100755
index e65bd8f..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-from resource_management.core.resources.system import Execute
-from slave import slave
-
-
-class Elasticsearch(Script):
- def install(self, env):
- import params
- env.set_params(params)
- print 'Install the Slave'
- Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
- Execute("echo \"[elasticsearch-2.x]\n"
- "name=Elasticsearch repository for 2.x packages\n"
- "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n"
- "gpgcheck=1\n"
- "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
- "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo")
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None, config_dir=None):
- import params
- env.set_params(params)
- slave()
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- stop_cmd = format("service elasticsearch stop")
- print 'Stop the Slave'
- Execute(stop_cmd)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env)
- start_cmd = format("service elasticsearch start")
- print 'Start the Slave'
- Execute(start_cmd)
-
- def status(self, env):
- import params
- env.set_params(params)
- status_cmd = format("service elasticsearch status")
- print 'Status of the Slave'
- Execute(status_cmd)
-
- def restart(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- restart_cmd = format("service elasticsearch restart")
- print 'Restarting the Slave'
- Execute(restart_cmd)
-
-
-if __name__ == "__main__":
- Elasticsearch().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
deleted file mode 100755
index c3e9169..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
-data_dir = config['configurations']['elastic-sysconfig']['data_dir']
-work_dir = config['configurations']['elastic-sysconfig']['work_dir']
-conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
-heap_size = config['configurations']['elastic-sysconfig']['heap_size']
-max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
-max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
-
-elastic_user = config['configurations']['elastic-env']['elastic_user']
-user_group = config['configurations']['elastic-env']['user_group']
-log_dir = config['configurations']['elastic-env']['elastic_log_dir']
-pid_dir = '/var/run/elasticsearch'
-pid_file = '/var/run/elasticsearch/elasticsearch.pid'
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-elastic_env_sh_template = config['configurations']['elastic-env']['content']
-sysconfig_template = config['configurations']['elastic-sysconfig']['content']
-
-cluster_name = config['configurations']['elastic-site']['cluster_name']
-zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
-
-path_data = config['configurations']['elastic-site']['path_data']
-http_port = config['configurations']['elastic-site']['http_port']
-transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port']
-
-recover_after_time = config['configurations']['elastic-site']['recover_after_time']
-gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
-expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes']
-discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled']
-index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
-index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size']
-index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval']
-indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
-index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards']
-index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas']
-indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size']
-bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall']
-threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size']
-cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
-cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low']
-cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled']
-cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high']
-indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size']
-indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping']
-threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size']
-
-discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout']
-discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval']
-discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout']
-discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries']
-
-network_host = config['configurations']['elastic-site']['network_host']
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
deleted file mode 100755
index ef9f6dd..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/properties_config.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-
-
-def properties_inline_template(configurations):
- return InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
-{% endfor %}''', configurations_dict=configurations)
-
-
-def properties_config(filename, configurations=None, conf_dir=None,
- mode=None, owner=None, group=None, brokerid=None):
- config_content = properties_inline_template(configurations)
- File(format("{conf_dir}/{filename}"), content=config_content, owner=owner,
- group=group, mode=mode)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
deleted file mode 100755
index 9615d83..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/service_check.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from __future__ import print_function
-
-import sys
-
-from resource_management.libraries.script import Script
-from resource_management.core.resources.system import Execute
-import subprocess
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- doc = '{"name": "Ambari Smoke test"}'
- index = "ambari_smoke_test"
-
- print("Running Elastic search service check", file=sys.stdout)
-
- # Make sure the service is actually up. We can live without everything allocated.
- # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
- # also hit ES before things are green.
- host = "localhost:9200"
- Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
- logoutput=True,
- tries=6,
- try_sleep=20
- )
-
- # Put a document into a new index.
-
- Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True)
-
- # Retrieve the document. Use subprocess because we actually need the results here.
- cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index)
- proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_retrieve = stdout
- print("Retrieval response is: %s" % response_retrieve)
- expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
- % (index, doc)
-
- # Delete the index
- cmd_delete = "curl -XDELETE '%s/%s'" % (host, index)
- proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- (stdout, stderr) = proc.communicate()
- response_delete = stdout
- print("Delete index response is: %s" % response_retrieve)
- expected_delete = '{"acknowledged":true}'
-
- if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
- print("Smoke test able to communicate with Elasticsearch")
- else:
- print("Elasticsearch service unable to retrieve document.")
- sys.exit(1)
-
- exit(0)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
deleted file mode 100755
index a134160..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-
-
-def slave():
- import params
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
-
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
-
- Directory(directories,
- create_parents=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_user,
- cd_access="a"
- )
-
- File("{}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- configurations = params.config['configurations']['elastic-site']
-
- File("{}/elasticsearch.yml".format(params.conf_dir),
- content=Template(
- "elasticsearch.slave.yaml.j2",
- configurations=configurations),
- owner=params.elastic_user,
- group=params.elastic_user
- )
-
- print "Master sysconfig: /etc/sysconfig/elasticsearch"
- File(format("/etc/sysconfig/elasticsearch"),
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
deleted file mode 100755
index 9cfb5cf..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/status_params.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script import Script
-
-config = Script.get_config()
-
-elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
-elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
deleted file mode 100755
index a9de018..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
+++ /dev/null
@@ -1,84 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- multicast:
- enabled: {{discovery_zen_ping_multicast_enabled}}
- unicast:
- hosts: "{{zen_discovery_ping_unicast_hosts}}"
-
-node:
- data: false
- master: true
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http.cors.enabled: true
-
-port: {{http_port}}
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-
-index:
- number_of_shards: {{index_number_of_shards}}
- merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
- translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
- refresh_interval: {{index_refresh_interval}}
- number_of_replicas: {{index_number_of_replicas}}
-
-indices:
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
- cluster:
- send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
-
-bootstrap.mlockall: {{bootstrap_mlockall}}
-
-threadpool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
deleted file mode 100755
index e88fc5f..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2
+++ /dev/null
@@ -1,84 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-cluster:
- name: {{cluster_name}}
- routing:
- allocation.node_concurrent_recoveries: {{cluster_routing_allocation_node_concurrent_recoveries}}
- allocation.disk.watermark.low: {{cluster_routing_allocation_disk_watermark_low}}
- allocation.disk.threshold_enabled: {{cluster_routing_allocation_disk_threshold_enabled}}
- allocation.disk.watermark.high: {{cluster_routing_allocation_disk_watermark_high}}
-
-discovery:
- zen:
- ping:
- multicast:
- enabled: {{discovery_zen_ping_multicast_enabled}}
- unicast:
- hosts: "{{zen_discovery_ping_unicast_hosts}}"
-
-node:
- data: true
- master: false
- name: {{hostname}}
-path:
- data: {{path_data}}
-
-http.cors.enabled: true
-
-port: {{http_port}}
-
-transport:
- tcp:
- port: {{transport_tcp_port}}
-
-gateway:
- recover_after_data_nodes: {{gateway_recover_after_data_nodes}}
- recover_after_time: {{recover_after_time}}
- expected_data_nodes: {{expected_data_nodes}}
-
-index:
- number_of_shards: {{index_number_of_shards}}
- merge.scheduler.max_thread_count: {{index_merge_scheduler_max_thread_count}}
- translog.flush_threshold_size: {{index_translog_flush_threshold_size}}
- refresh_interval: {{index_refresh_interval}}
- number_of_replicas: {{index_number_of_replicas}}
-
-indices:
- memory:
- index_buffer_size: {{indices_memory_index_buffer_size}}
- store.throttle.type: {{indices_memory_index_store_throttle_type}}
- fielddata:
- cache.size: {{indices_fielddata_cache_size}}
- cluster:
- send_refresh_mapping: {{indices_cluster_send_refresh_mapping}}
-
-bootstrap.mlockall: {{bootstrap_mlockall}}
-
-threadpool:
- bulk:
- queue_size: {{threadpool_bulk_queue_size}}
- index:
- queue_size: {{threadpool_index_queue_size}}
-
-discovery.zen.ping_timeout: {{discovery_zen_ping_timeout}}
-discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}}
-discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}}
-discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}}
-
-network.host: {{network_host}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json b/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
deleted file mode 100755
index 130d018..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/ELASTICSEARCH/2.3.3/role_command_order.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "ELASTICSEARCH_SERVICE_CHECK-SERVICE_CHECK" : ["ES_MASTER-START", "ES_SLAVE-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
deleted file mode 100755
index 3a1c8f7..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration supports_final="true">
- <property>
- <name>metron_home</name>
- <value>/usr/metron/0.2.0BETA</value>
- <description>Metron home directory</description>
- <display-name>Metron home</display-name>
- </property>
- <property>
- <name>metron_apps_hdfs_dir</name>
- <value>/apps/metron</value>
- <description>Metron apps HDFS dir</description>
- <display-name>Metron apps HDFS dir</display-name>
- </property>
- <property>
- <name>metron_user</name>
- <value>metron</value>
- <property-type>USER</property-type>
- <description>The user for Metron</description>
- <display-name>Metron User</display-name>
- </property>
- <property>
- <name>metron_group</name>
- <value>metron</value>
- <description>The group for Metron</description>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
deleted file mode 100755
index d2dce5a..0000000
--- a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
+++ /dev/null
@@ -1,105 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>INDEXING</name>
- <displayName>Indexing</displayName>
- <comment>Indexing</comment>
- <version>0.2.0BETA</version>
-
- <components>
- <component>
- <name>INDEXING_MASTER</name>
- <displayName>Indexing Master-Eligible Node</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>KAFKA/KAFKA_BROKER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/indexing_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>metron-common</name>
- </package>
- <package>
- <name>metron-indexing</name>
- </package>
- <package>
- <name>metron-elasticsearch</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- <service>KAFKA</service>
- <service>STORM</service>
- <service>ZOOKEEPER</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>metron-indexing</config-type>
- </configuration-dependencies>
-
- <quickLinksConfigurations>
- <quickLinksConfiguration>
- <fileName>quicklinks.json</fileName>
- <default>true</default>
- </quickLinksConfiguration>
- </quickLinksConfigurations>
- </service>
- </services>
-</metainfo>
[6/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
new file mode 100755
index 0000000..c546e2c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- kibana.yml -->
+ <property>
+ <name>content</name>
+ <display-name>kibana.yml template</display-name>
+ <description>This is the jinja template for kibana.yml file</description>
+ <value>
+# Kibana is served by a back end server. This controls which port to use.
+server.port: {{ kibana_port }}
+
+# The host to bind the server to.
+# server.host: "0.0.0.0"
+
+# If you are running kibana behind a proxy, and want to mount it at a path,
+# specify that path here. The basePath can't end in a slash.
+# server.basePath: ""
+
+# The maximum payload size in bytes on incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The Elasticsearch instance to use for all your queries.
+elasticsearch.url: {{ es_url }}
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+# elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+# kibana.index: ".kibana"
+
+# The default application to load.
+# kibana.defaultAppId: "discover"
+
+# If your Elasticsearch is protected with basic auth, these are the user credentials
+# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied through
+# the Kibana server)
+# elasticsearch.username: "user"
+# elasticsearch.password: "pass"
+
+# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
+# server.ssl.cert: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+
+# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
+# elasticsearch.ssl.cert: /path/to/your/client.crt
+# elasticsearch.ssl.key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsearch instance, put
+# the path of the pem file here.
+# elasticsearch.ssl.ca: /path/to/your/CA.pem
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+# elasticsearch.ssl.verify: true
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+# elasticsearch.requestTimeout: 30000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+# elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# elasticsearch.startupTimeout: 5000
+
+# Set the path to where you would like the process id file to be created.
+# pid.file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+logging.dest: {{ log_dir }}/kibana.log
+
+# Set this to true to suppress all logging output.
+# logging.silent: false
+
+# Set this to true to suppress all logging output except for error messages.
+# logging.quiet: false
+
+# Set this to true to log all events, including system usage information and all requests.
+# logging.verbose: false
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
new file mode 100755
index 0000000..f59109c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/metainfo.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KIBANA</name>
+ <displayName>Kibana</displayName>
+ <comment>Kibana Dashboard</comment>
+ <version>4.5.1</version>
+ <components>
+ <component>
+ <name>KIBANA_MASTER</name>
+ <displayName>Kibana Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/kibana_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>LOAD_TEMPLATE</name>
+ <background>false</background>
+ <commandScript>
+ <script>scripts/kibana_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-4.5.1</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <configuration-dependencies>
+ <config-type>kibana-env</config-type>
+ <config-type>kibana-site</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
new file mode 100755
index 0000000..8d2bad8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
new file mode 100755
index 0000000..8327eb8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p
@@ -0,0 +1,1539 @@
+(lp1
+(dp2
+V_score
+p3
+F1
+sV_type
+p4
+Vvisualization
+p5
+sV_id
+p6
+VWeb-Request-Type
+p7
+sV_source
+p8
+(dp9
+VvisState
+p10
+V{"title":"Web Request Type","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"method","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p11
+sVdescription
+p12
+V
+sVtitle
+p13
+VWeb Request Type
+p14
+sVuiStateJSON
+p15
+V{}
+p16
+sVversion
+p17
+I1
+sVsavedSearchId
+p18
+Vweb-search
+p19
+sVkibanaSavedObjectMeta
+p20
+(dp21
+VsearchSourceJSON
+p22
+V{"filter":[]}
+p23
+sssV_index
+p24
+V.kibana
+p25
+sa(dp26
+V_score
+p27
+F1
+sV_type
+p28
+Vvisualization
+p29
+sV_id
+p30
+VTop-Snort-Alerts-by-Source
+p31
+sV_source
+p32
+(dp33
+VvisState
+p34
+V{"title":"Top Snort Alerts by Source","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Source IP"}}],"listeners":{}}
+p35
+sVdescription
+p36
+V
+sVtitle
+p37
+VTop Snort Alerts by Source
+p38
+sVuiStateJSON
+p39
+V{}
+p40
+sVversion
+p41
+I1
+sVkibanaSavedObjectMeta
+p42
+(dp43
+VsearchSourceJSON
+p44
+V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p45
+sssV_index
+p46
+V.kibana
+p47
+sa(dp48
+V_score
+p49
+F1
+sV_type
+p50
+Vvisualization
+p51
+sV_id
+p52
+VWelcome
+p53
+sV_source
+p54
+(dp55
+VvisState
+p56
+V{"title":"Welcome to Apache Metron","type":"markdown","params":{"markdown":"This dashboard enables the validation of Apache Metron and the end-to-end functioning of its default sensor suite. The default sensor suite includes [Snort](https://www.snort.org/), [Bro](https://www.bro.org/), and [YAF](https://tools.netsa.cert.org/yaf/). One of Apache Metron's primary goals is to simplify the onboarding of additional sources of telemetry. In a production deployment these default sensors should be replaced with ones applicable to the target environment.\u005cn\u005cnApache Metron enables disparate sources of telemetry to all be viewed under a 'single pane of glass.' Telemetry from each of the default sensors can be searched, aggregated, summarized, and viewed within this dashboard. This dashboard should be used as a springboard upon which to create your own customized dashboards.\u005cn\u005cnThe panels below highlight the volume and variety of events that are currently being consumed
by Apache Metron."},"aggs":[],"listeners":{}}
+p57
+sVdescription
+p58
+V
+sVtitle
+p59
+VWelcome to Apache Metron
+p60
+sVuiStateJSON
+p61
+V{}
+p62
+sVversion
+p63
+I1
+sVkibanaSavedObjectMeta
+p64
+(dp65
+VsearchSourceJSON
+p66
+V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
+p67
+sssV_index
+p68
+V.kibana
+p69
+sa(dp70
+V_score
+p71
+F1
+sV_type
+p72
+Vsearch
+p73
+sV_id
+p74
+Vsnort-search
+p75
+sV_source
+p76
+(dp77
+Vsort
+p78
+(lp79
+Vtimestamp
+p80
+aVdesc
+p81
+asVhits
+p82
+I0
+sVdescription
+p83
+V
+sVtitle
+p84
+VSnort Alerts
+p85
+sVversion
+p86
+I1
+sVkibanaSavedObjectMeta
+p87
+(dp88
+VsearchSourceJSON
+p89
+V{"index":"snort*","query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
+p90
+ssVcolumns
+p91
+(lp92
+Vmsg
+p93
+aVsig_id
+p94
+aVip_src_addr
+p95
+aVip_src_port
+p96
+aVip_dst_addr
+p97
+aVip_dst_port
+p98
+assV_index
+p99
+V.kibana
+p100
+sa(dp101
+V_score
+p102
+F1
+sV_type
+p103
+Vsearch
+p104
+sV_id
+p105
+Vyaf-search
+p106
+sV_source
+p107
+(dp108
+Vsort
+p109
+(lp110
+Vtimestamp
+p111
+aVdesc
+p112
+asVhits
+p113
+I0
+sVdescription
+p114
+V
+sVtitle
+p115
+VYAF
+p116
+sVversion
+p117
+I1
+sVkibanaSavedObjectMeta
+p118
+(dp119
+VsearchSourceJSON
+p120
+V{"index":"yaf*","filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647},"query":{"query_string":{"query":"*","analyze_wildcard":true}}}
+p121
+ssVcolumns
+p122
+(lp123
+Vip_src_addr
+p124
+aVip_src_port
+p125
+aVip_dst_addr
+p126
+aVip_dst_port
+p127
+aVprotocol
+p128
+aVduration
+p129
+aVpkt
+p130
+assV_index
+p131
+V.kibana
+p132
+sa(dp133
+V_score
+p134
+F1
+sV_type
+p135
+Vconfig
+p136
+sV_id
+p137
+V4.5.1
+p138
+sV_source
+p139
+(dp140
+VbuildNum
+p141
+I9892
+sVdefaultIndex
+p142
+Vbro*
+p143
+ssV_index
+p144
+V.kibana
+p145
+sa(dp146
+V_score
+p147
+F1
+sV_type
+p148
+Vindex-pattern
+p149
+sV_id
+p150
+Vbro*
+p151
+sV_source
+p152
+(dp153
+Vfields
+p154
+V[{"name":"TTLs","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qclass_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"bro_timestamp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"answers","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"prot
ocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"host","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":1,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"AA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"method","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"query","type":"string","count":0,"s
cripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_mime_types","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"RA","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"RD","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"orig_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false
,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"response_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype_name","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_code","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,
"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rejected","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"qtype","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"ana
lyzed":false,"doc_values":true},{"name":"uid","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"trans_depth","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"Z","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enr
ichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"user_agent","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"qclass","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"resp_fuids","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"request_body_len","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uri","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rcode_name","type":"string","coun
t":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"TC","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"referrer","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"status_msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":1,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":2,"scripted":false,"indexed":false,"analyzed":false,"doc_values":
false}]
+p155
+sVtimeFieldName
+p156
+Vtimestamp
+p157
+sVtitle
+p158
+Vbro*
+p159
+ssV_index
+p160
+V.kibana
+p161
+sa(dp162
+V_score
+p163
+F1
+sV_type
+p164
+Vvisualization
+p165
+sV_id
+p166
+VFlow-Duration
+p167
+sV_source
+p168
+(dp169
+VvisState
+p170
+V{"title":"Flow Duration","type":"area","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"smoothLines":false,"scale":"linear","interpolate":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"histogram","schema":"segment","params":{"field":"duration","interval":10,"extended_bounds":{},"customLabel":"Flow Duration (seconds)"}}],"listeners":{}}
+p171
+sVdescription
+p172
+V
+sVtitle
+p173
+VFlow Duration
+p174
+sVuiStateJSON
+p175
+V{"vis":{"legendOpen":false}}
+p176
+sVversion
+p177
+I1
+sVkibanaSavedObjectMeta
+p178
+(dp179
+VsearchSourceJSON
+p180
+V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p181
+sssV_index
+p182
+V.kibana
+p183
+sa(dp184
+V_score
+p185
+F1
+sV_type
+p186
+Vvisualization
+p187
+sV_id
+p188
+VEvents
+p189
+sV_source
+p190
+(dp191
+VvisState
+p192
+V{"title":"Events","type":"histogram","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"scale":"linear","mode":"stacked","times":[],"addTimeMarker":false,"defaultYExtents":false,"setYExtents":false,"yAxis":{}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"date_histogram","schema":"segment","params":{"field":"timestamp","interval":"auto","customInterval":"2h","min_doc_count":1,"extended_bounds":{}}},{"id":"3","type":"terms","schema":"group","params":{"field":"source:type","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p193
+sVdescription
+p194
+V
+sVtitle
+p195
+VEvents
+p196
+sVuiStateJSON
+p197
+V{"vis":{"legendOpen":false}}
+p198
+sVversion
+p199
+I1
+sVkibanaSavedObjectMeta
+p200
+(dp201
+VsearchSourceJSON
+p202
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p203
+sssV_index
+p204
+V.kibana
+p205
+sa(dp206
+V_score
+p207
+F1
+sV_type
+p208
+Vvisualization
+p209
+sV_id
+p210
+VWeb-Request-Header
+p211
+sV_source
+p212
+(dp213
+VvisState
+p214
+V{"title":"Web Request Header","type":"markdown","params":{"markdown":"The [Bro Network Security Monitor](https://www.bro.org/) is extracting application-level information from raw network packets. In this example, Bro is extracting HTTP(S) requests being made over the network. "},"aggs":[],"listeners":{}}
+p215
+sVdescription
+p216
+V
+sVtitle
+p217
+VWeb Request Header
+p218
+sVuiStateJSON
+p219
+V{}
+p220
+sVversion
+p221
+I1
+sVkibanaSavedObjectMeta
+p222
+(dp223
+VsearchSourceJSON
+p224
+V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p225
+sssV_index
+p226
+V.kibana
+p227
+sa(dp228
+V_score
+p229
+F1
+sV_type
+p230
+Vvisualization
+p231
+sV_id
+p232
+VTop-Alerts-By-Host
+p233
+sV_source
+p234
+(dp235
+VvisState
+p236
+V{"title":"New Visualization","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_src_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Source"}},{"id":"3","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":5,"order":"desc","orderBy":"1","customLabel":"Destination"}}],"listeners":{}}
+p237
+sVdescription
+p238
+V
+sVtitle
+p239
+VTop Alerts By Host
+p240
+sVuiStateJSON
+p241
+V{}
+p242
+sVversion
+p243
+I1
+sVsavedSearchId
+p244
+Vsnort-search
+p245
+sVkibanaSavedObjectMeta
+p246
+(dp247
+VsearchSourceJSON
+p248
+V{"filter":[]}
+p249
+sssV_index
+p250
+V.kibana
+p251
+sa(dp252
+V_score
+p253
+F1
+sV_type
+p254
+Vvisualization
+p255
+sV_id
+p256
+VYAF-Flow(s)
+p257
+sV_source
+p258
+(dp259
+VvisState
+p260
+V{"title":"YAF Flows","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
+p261
+sVdescription
+p262
+V
+sVtitle
+p263
+VYAF Flows
+p264
+sVuiStateJSON
+p265
+V{}
+p266
+sVversion
+p267
+I1
+sVkibanaSavedObjectMeta
+p268
+(dp269
+VsearchSourceJSON
+p270
+V{"index":"yaf*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p271
+sssV_index
+p272
+V.kibana
+p273
+sa(dp274
+V_score
+p275
+F1
+sV_type
+p276
+Vvisualization
+p277
+sV_id
+p278
+VTop-DNS-Query
+p279
+sV_source
+p280
+(dp281
+VvisState
+p282
+V{"title":"Top DNS Query","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p283
+sVdescription
+p284
+V
+sVtitle
+p285
+VTop DNS Query
+p286
+sVuiStateJSON
+p287
+V{}
+p288
+sVversion
+p289
+I1
+sVkibanaSavedObjectMeta
+p290
+(dp291
+VsearchSourceJSON
+p292
+V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p293
+sssV_index
+p294
+V.kibana
+p295
+sa(dp296
+V_score
+p297
+F1
+sV_type
+p298
+Vvisualization
+p299
+sV_id
+p300
+VTotal-Events
+p301
+sV_source
+p302
+(dp303
+VvisState
+p304
+V{"title":"Event Count","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{"customLabel":"Events"}}],"listeners":{}}
+p305
+sVdescription
+p306
+V
+sVtitle
+p307
+VEvent Count
+p308
+sVuiStateJSON
+p309
+V{}
+p310
+sVversion
+p311
+I1
+sVkibanaSavedObjectMeta
+p312
+(dp313
+VsearchSourceJSON
+p314
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p315
+sssV_index
+p316
+V.kibana
+p317
+sa(dp318
+V_score
+p319
+F1
+sV_type
+p320
+Vvisualization
+p321
+sV_id
+p322
+VEvent-Types
+p323
+sV_source
+p324
+(dp325
+VvisState
+p326
+V{"title":"Event Sources","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"source:type","size":10,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p327
+sVdescription
+p328
+V
+sVtitle
+p329
+VEvent Sources
+p330
+sVuiStateJSON
+p331
+V{}
+p332
+sVversion
+p333
+I1
+sVkibanaSavedObjectMeta
+p334
+(dp335
+VsearchSourceJSON
+p336
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p337
+sssV_index
+p338
+V.kibana
+p339
+sa(dp340
+V_score
+p341
+F1
+sV_type
+p342
+Vvisualization
+p343
+sV_id
+p344
+VUnique-Location(s)
+p345
+sV_source
+p346
+(dp347
+VvisState
+p348
+V{"title":"Geo-IP Locations","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"enrichments:geo:ip_src_addr:locID","customLabel":"Unique Location(s)"}}],"listeners":{}}
+p349
+sVdescription
+p350
+V
+sVtitle
+p351
+VGeo-IP Locations
+p352
+sVuiStateJSON
+p353
+V{}
+p354
+sVversion
+p355
+I1
+sVkibanaSavedObjectMeta
+p356
+(dp357
+VsearchSourceJSON
+p358
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p359
+sssV_index
+p360
+V.kibana
+p361
+sa(dp362
+V_score
+p363
+F1
+sV_type
+p364
+Vvisualization
+p365
+sV_id
+p366
+VSnort-Header
+p367
+sV_source
+p368
+(dp369
+VvisState
+p370
+V{"title":"Snort","type":"markdown","params":{"markdown":"[Snort](https://www.snort.org/) is a Network Intrusion Detection System (NIDS) that is being used to generate alerts identifying known bad events. Snort relies on a fixed set of rules that act as signatures for identifying abnormal events."},"aggs":[],"listeners":{}}
+p371
+sVdescription
+p372
+V
+sVtitle
+p373
+VSnort
+p374
+sVuiStateJSON
+p375
+V{}
+p376
+sVversion
+p377
+I1
+sVkibanaSavedObjectMeta
+p378
+(dp379
+VsearchSourceJSON
+p380
+V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p381
+sssV_index
+p382
+V.kibana
+p383
+sa(dp384
+V_score
+p385
+F1
+sV_type
+p386
+Vdashboard
+p387
+sV_id
+p388
+VMetron-Dashboard
+p389
+sV_source
+p390
+(dp391
+Vhits
+p392
+I0
+sVtimeRestore
+p393
+I00
+sVdescription
+p394
+V
+sVtitle
+p395
+VMetron Dashboard
+p396
+sVuiStateJSON
+p397
+V{"P-23":{"spy":{"mode":{"name":null,"fill":false}}},"P-34":{"vis":{"legendOpen":false}}}
+p398
+sVpanelsJSON
+p399
+V[{"col":1,"id":"Welcome","panelIndex":30,"row":1,"size_x":11,"size_y":2,"type":"visualization"},{"col":1,"id":"Total-Events","panelIndex":6,"row":3,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Events","panelIndex":16,"row":3,"size_x":8,"size_y":4,"type":"visualization"},{"col":1,"id":"Event-Types","panelIndex":15,"row":5,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Location-Header","panelIndex":24,"row":7,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Unique-Location(s)","panelIndex":23,"row":9,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"id":"Flow-Locations","panelIndex":32,"row":7,"size_x":8,"size_y":6,"type":"visualization"},{"col":1,"id":"Country","panelIndex":8,"row":11,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flows-Header","panelIndex":27,"row":13,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"YAF-Flow(s)","panelIndex":21,"row":15,"size_x":3,"size_y":2,"type":"visualization"},{"col"
:4,"columns":["ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port","protocol","duration","pkt"],"id":"yaf-search","panelIndex":20,"row":13,"size_x":8,"size_y":6,"sort":["duration","desc"],"type":"search"},{"col":1,"id":"Flow-Duration","panelIndex":31,"row":17,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Snort-Header","panelIndex":25,"row":19,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["msg","sig_id","ip_src_addr","ip_src_port","ip_dst_addr","ip_dst_port"],"id":"snort-search","panelIndex":3,"row":19,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"Snort-Alert-Types","panelIndex":10,"row":21,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Top-Alerts-By-Host","panelIndex":19,"row":23,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Header","panelIndex":26,"row":25,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["method","host","uri","referrer","user_agent","i
p_src_addr","ip_dst_addr"],"id":"web-search","panelIndex":4,"row":25,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"HTTP(S)-Requests","panelIndex":17,"row":27,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"DNS-Requests-Header","panelIndex":29,"row":31,"size_x":3,"size_y":2,"type":"visualization"},{"col":4,"columns":["query","qtype_name","answers","ip_src_addr","ip_dst_addr"],"id":"dns-search","panelIndex":5,"row":31,"size_x":8,"size_y":6,"sort":["timestamp","desc"],"type":"search"},{"col":1,"id":"DNS-Request(s)","panelIndex":14,"row":33,"size_x":3,"size_y":2,"type":"visualization"},{"col":1,"id":"Web-Request-Type","panelIndex":33,"row":29,"size_x":3,"size_y":2,"type":"visualization"}]
+p400
+sVoptionsJSON
+p401
+V{"darkTheme":false}
+p402
+sVversion
+p403
+I1
+sVkibanaSavedObjectMeta
+p404
+(dp405
+VsearchSourceJSON
+p406
+V{"filter":[{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}}}]}
+p407
+sssV_index
+p408
+V.kibana
+p409
+sa(dp410
+V_score
+p411
+F1
+sV_type
+p412
+Vvisualization
+p413
+sV_id
+p414
+VSnort-Alert-Types
+p415
+sV_source
+p416
+(dp417
+VvisState
+p418
+V{"title":"Snort Alert Types","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"cardinality","schema":"metric","params":{"field":"sig_id","customLabel":"Alert Type(s)"}}],"listeners":{}}
+p419
+sVdescription
+p420
+V
+sVtitle
+p421
+VSnort Alert Types
+p422
+sVuiStateJSON
+p423
+V{}
+p424
+sVversion
+p425
+I1
+sVkibanaSavedObjectMeta
+p426
+(dp427
+VsearchSourceJSON
+p428
+V{"index":"snort*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p429
+sssV_index
+p430
+V.kibana
+p431
+sa(dp432
+V_score
+p433
+F1
+sV_type
+p434
+Vvisualization
+p435
+sV_id
+p436
+VFrequent-DNS-Queries
+p437
+sV_source
+p438
+(dp439
+VvisState
+p440
+V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p441
+sVdescription
+p442
+V
+sVtitle
+p443
+VFrequent DNS Requests
+p444
+sVuiStateJSON
+p445
+V{}
+p446
+sVversion
+p447
+I1
+sVkibanaSavedObjectMeta
+p448
+(dp449
+VsearchSourceJSON
+p450
+V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p451
+sssV_index
+p452
+V.kibana
+p453
+sa(dp454
+V_score
+p455
+F1
+sV_type
+p456
+Vvisualization
+p457
+sV_id
+p458
+VLocation-Header
+p459
+sV_source
+p460
+(dp461
+VvisState
+p462
+V{"title":"Enrichment","type":"markdown","params":{"markdown":"Apache Metron can perform real-time enrichment of telemetry data as it is consumed. To highlight this feature, all of the IP address fields collected from the default sensor suite were used to perform geo-ip lookups. This data was then used to pinpoint each location on the map."},"aggs":[],"listeners":{}}
+p463
+sVdescription
+p464
+V
+sVtitle
+p465
+VEnrichment
+p466
+sVuiStateJSON
+p467
+V{}
+p468
+sVversion
+p469
+I1
+sVkibanaSavedObjectMeta
+p470
+(dp471
+VsearchSourceJSON
+p472
+V{"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p473
+sssV_index
+p474
+V.kibana
+p475
+sa(dp476
+V_score
+p477
+F1
+sV_type
+p478
+Vsearch
+p479
+sV_id
+p480
+Vweb-search
+p481
+sV_source
+p482
+(dp483
+Vsort
+p484
+(lp485
+Vtimestamp
+p486
+aVdesc
+p487
+asVhits
+p488
+I0
+sVdescription
+p489
+V
+sVtitle
+p490
+VWeb Requests
+p491
+sVversion
+p492
+I1
+sVkibanaSavedObjectMeta
+p493
+(dp494
+VsearchSourceJSON
+p495
+V{"index":"bro*","query":{"query_string":{"query":"protocol: http OR protocol: https","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
+p496
+ssVcolumns
+p497
+(lp498
+Vmethod
+p499
+aVhost
+p500
+aVuri
+p501
+aVreferrer
+p502
+aVip_src_addr
+p503
+aVip_dst_addr
+p504
+assV_index
+p505
+V.kibana
+p506
+sa(dp507
+V_score
+p508
+F1
+sV_type
+p509
+Vindex-pattern
+p510
+sV_id
+p511
+Vsnort*
+p512
+sV_source
+p513
+(dp514
+Vfields
+p515
+V[{"name":"msg","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dgmlen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpack","type":"string","count":0,"scripted":false,"indexed":true,"analyze
d":true,"doc_values":false},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scr
ipted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethlen","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threat:triage:level","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type"
:"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_rev","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"ethsrc
","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpseq","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tcpwindow","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tos","type":"n
umber","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:latitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ethdst","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"
is_alert","type":"boolean","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ttl","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iplen","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_id","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sig_generator","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_src_addr:city","t
ype":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}]
+p516
+sVtimeFieldName
+p517
+Vtimestamp
+p518
+sVtitle
+p519
+Vsnort*
+p520
+ssV_index
+p521
+V.kibana
+p522
+sa(dp523
+V_score
+p524
+F1
+sV_type
+p525
+Vindex-pattern
+p526
+sV_id
+p527
+Vyaf*
+p528
+sV_source
+p529
+(dp530
+Vfields
+p531
+V[{"name":"enrichments:geo:ip_dst_addr:location_point","type":"geo_point","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"isn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentjoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"dp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"protocol","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"rpkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"original_string","type":"strin
g","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"adapter:threatinteladapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:geoadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"tag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"app","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"oct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"enrichmentsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:city","type":"string","count":0,"sc
ripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"start_time","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"riflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"proto","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:threatinteladapter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_source","type":"_source","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"enrichments:geo:ip_dst_addr:country","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:locID","type":"string","
count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"iflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_index","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"ip_dst_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:dmaCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatinteljoinbolt:joiner:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"uflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichmentsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:latitude","type":
"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"duration","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"source:type","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_dst_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"pkt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"adapter:hostfromjsonlistadapter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ruflags","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"roct","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"sip","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_value
s":true},{"name":"sp","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_addr","type":"ip","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtag","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:end:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:longitude","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"timestamp","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end-reason","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":true,"doc_values":false},{"name":"risn","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"end_time","type":"date","count"
:0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"enrichments:geo:ip_dst_addr:postalCode","type":"string","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"rtt","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"ip_src_port","type":"number","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"threatintelsplitterbolt:splitter:begin:ts","type":"date","count":0,"scripted":false,"indexed":true,"analyzed":false,"doc_values":true},{"name":"_id","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_type","type":"string","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false},{"name":"_score","type":"number","count":0,"scripted":false,"indexed":false,"analyzed":false,"doc_values":false}]
+p532
+sVtimeFieldName
+p533
+Vtimestamp
+p534
+sVtitle
+p535
+Vyaf*
+p536
+ssV_index
+p537
+V.kibana
+p538
+sa(dp539
+V_score
+p540
+F1
+sV_type
+p541
+Vvisualization
+p542
+sV_id
+p543
+VDNS-Request(s)
+p544
+sV_source
+p545
+(dp546
+VvisState
+p547
+V{"title":"DNS Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
+p548
+sVdescription
+p549
+V
+sVtitle
+p550
+VDNS Requests
+p551
+sVuiStateJSON
+p552
+V{}
+p553
+sVversion
+p554
+I1
+sVsavedSearchId
+p555
+Vdns-search
+p556
+sVkibanaSavedObjectMeta
+p557
+(dp558
+VsearchSourceJSON
+p559
+V{"filter":[]}
+p560
+sssV_index
+p561
+V.kibana
+p562
+sa(dp563
+V_score
+p564
+F1
+sV_type
+p565
+Vvisualization
+p566
+sV_id
+p567
+VHTTP(S)-Requests
+p568
+sV_source
+p569
+(dp570
+VvisState
+p571
+V{"title":"Web Requests","type":"metric","params":{"handleNoResults":true,"fontSize":60},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}}],"listeners":{}}
+p572
+sVdescription
+p573
+V
+sVtitle
+p574
+VWeb Requests
+p575
+sVuiStateJSON
+p576
+V{}
+p577
+sVversion
+p578
+I1
+sVsavedSearchId
+p579
+Vweb-search
+p580
+sVkibanaSavedObjectMeta
+p581
+(dp582
+VsearchSourceJSON
+p583
+V{"filter":[]}
+p584
+sssV_index
+p585
+V.kibana
+p586
+sa(dp587
+V_score
+p588
+F1
+sV_type
+p589
+Vsearch
+p590
+sV_id
+p591
+Vdns-search
+p592
+sV_source
+p593
+(dp594
+Vsort
+p595
+(lp596
+Vtimestamp
+p597
+aVdesc
+p598
+asVhits
+p599
+I0
+sVdescription
+p600
+V
+sVtitle
+p601
+VDNS Requests
+p602
+sVversion
+p603
+I1
+sVkibanaSavedObjectMeta
+p604
+(dp605
+VsearchSourceJSON
+p606
+V{"index":"bro*","query":{"query_string":{"query":"protocol: dns","analyze_wildcard":true}},"filter":[],"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}},"require_field_match":false,"fragment_size":2147483647}}
+p607
+ssVcolumns
+p608
+(lp609
+Vquery
+p610
+aVqtype_name
+p611
+aVanswers
+p612
+aVip_src_addr
+p613
+aVip_dst_addr
+p614
+assV_index
+p615
+V.kibana
+p616
+sa(dp617
+V_score
+p618
+F1
+sV_type
+p619
+Vvisualization
+p620
+sV_id
+p621
+VFlow-Locations
+p622
+sV_source
+p623
+(dp624
+VvisState
+p625
+V{"title":"New Visualization","type":"tile_map","params":{"mapType":"Scaled Circle Markers","isDesaturated":true,"addTooltip":true,"heatMaxZoom":16,"heatMinOpacity":0.1,"heatRadius":25,"heatBlur":15,"heatNormalizeData":true,"wms":{"enabled":false,"url":"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer","options":{"version":"1.3.0","layers":"0","format":"image/png","transparent":true,"attribution":"Maps provided by USGS","styles":""}}},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"geohash_grid","schema":"segment","params":{"field":"enrichments:geo:ip_dst_addr:location_point","autoPrecision":true,"precision":2}}],"listeners":{}}
+p626
+sVdescription
+p627
+V
+sVtitle
+p628
+VFlow Locations
+p629
+sVuiStateJSON
+p630
+V{}
+p631
+sVversion
+p632
+I1
+sVkibanaSavedObjectMeta
+p633
+(dp634
+VsearchSourceJSON
+p635
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p636
+sssV_index
+p637
+V.kibana
+p638
+sa(dp639
+V_score
+p640
+F1
+sV_type
+p641
+Vvisualization
+p642
+sV_id
+p643
+VUnusual-Referrers
+p644
+sV_source
+p645
+(dp646
+VvisState
+p647
+V{"title":"Unusual Referrers","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"significant_terms","schema":"bucket","params":{"field":"referrer","size":5,"customLabel":"Top 5 Unusual Referrers"}}],"listeners":{}}
+p648
+sVdescription
+p649
+V
+sVtitle
+p650
+VUnusual Referrers
+p651
+sVuiStateJSON
+p652
+V{}
+p653
+sVversion
+p654
+I1
+sVsavedSearchId
+p655
+Vweb-search
+p656
+sVkibanaSavedObjectMeta
+p657
+(dp658
+VsearchSourceJSON
+p659
+V{"filter":[]}
+p660
+sssV_index
+p661
+V.kibana
+p662
+sa(dp663
+V_score
+p664
+F1
+sV_type
+p665
+Vvisualization
+p666
+sV_id
+p667
+VFrequent-DNS-Requests
+p668
+sV_source
+p669
+(dp670
+VvisState
+p671
+V{"title":"Frequent DNS Requests","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"query","size":5,"order":"desc","orderBy":"1","customLabel":"DNS Query"}}],"listeners":{}}
+p672
+sVdescription
+p673
+V
+sVtitle
+p674
+VFrequent DNS Requests
+p675
+sVuiStateJSON
+p676
+V{}
+p677
+sVversion
+p678
+I1
+sVkibanaSavedObjectMeta
+p679
+(dp680
+VsearchSourceJSON
+p681
+V{"index":"bro*","query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p682
+sssV_index
+p683
+V.kibana
+p684
+sa(dp685
+V_score
+p686
+F1
+sV_type
+p687
+Vvisualization
+p688
+sV_id
+p689
+VCountry
+p690
+sV_source
+p691
+(dp692
+VvisState
+p693
+V{"title":"By Country","type":"pie","params":{"shareYAxis":true,"addTooltip":true,"addLegend":true,"isDonut":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"segment","params":{"field":"enrichments:geo:ip_src_addr:country","size":5,"order":"desc","orderBy":"1"}}],"listeners":{}}
+p694
+sVdescription
+p695
+V
+sVtitle
+p696
+VBy Country
+p697
+sVuiStateJSON
+p698
+V{}
+p699
+sVversion
+p700
+I1
+sVkibanaSavedObjectMeta
+p701
+(dp702
+VsearchSourceJSON
+p703
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p704
+sssV_index
+p705
+V.kibana
+p706
+sa(dp707
+V_score
+p708
+F1
+sV_type
+p709
+Vvisualization
+p710
+sV_id
+p711
+VTop-Destinations
+p712
+sV_source
+p713
+(dp714
+VvisState
+p715
+V{"title":"Top Destinations","type":"table","params":{"perPage":10,"showPartialRows":false,"showMeticsAtAllLevels":false},"aggs":[{"id":"1","type":"count","schema":"metric","params":{}},{"id":"2","type":"terms","schema":"bucket","params":{"field":"ip_dst_addr","size":10,"order":"desc","orderBy":"1","customLabel":"Destination IP"}}],"listeners":{}}
+p716
+sVdescription
+p717
+V
+sVtitle
+p718
+VTop Destinations
+p719
+sVuiStateJSON
+p720
+V{}
+p721
+sVversion
+p722
+I1
+sVkibanaSavedObjectMeta
+p723
+(dp724
+VsearchSourceJSON
+p725
+V{"index":["yaf*", "bro*", "snort*"],"query":{"query_string":{"query":"*","analyze_wildcard":true}},"filter":[]}
+p726
+sssV_index
+p727
+V.kibana
+p728
+sa(dp729
+V_score
+p730
+F1
+sV_type
+p731
+Vvisualization
+p732
+sV_id
+p733
+VDNS-Requests-Header
+p734
+sV_source
+p735
+(dp736
+VvisState
+p737
+V{"aggs":[],"listeners":{},"params":{"markdown":"[Bro](https://www.bro.org/) is extracting DNS requests and responses being made over the network. Understanding who is making those requests, the frequency, and types can provide a deep understanding of the actors present on the network."},"title":"DNS Requests","type":"markdown"}
+p738
+sVdescription
+p739
+V
+sVtitle
+p740
+VDNS Requests
+p741
+sVuiStateJSON
+p742
+V{}
+p743
+sVversion
+p744
+I1
+sVkibanaSavedObjectMeta
+p745
+(dp746
+VsearchSourceJSON
+p747
+V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
+p748
+sssV_index
+p749
+V.kibana
+p750
+sa(dp751
+V_score
+p752
+F1
+sV_type
+p753
+Vvisualization
+p754
+sV_id
+p755
+VYAF-Flows-Header
+p756
+sV_source
+p757
+(dp758
+VvisState
+p759
+V{"title":"YAF","type":"markdown","params":{"markdown":"[YAF](https://tools.netsa.cert.org/yaf/yaf.html) can be used to generate Netflow-like flow records. These flow records provide significant visibility of the actors communicating over the target network."},"aggs":[],"listeners":{}}
+p760
+sVdescription
+p761
+V
+sVtitle
+p762
+VYAF
+p763
+sVuiStateJSON
+p764
+V{}
+p765
+sVversion
+p766
+I1
+sVkibanaSavedObjectMeta
+p767
+(dp768
+VsearchSourceJSON
+p769
+V{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":[]}
+p770
+sssV_index
+p771
+V.kibana
+p772
+sa.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
new file mode 100755
index 0000000..f0903ac
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+import cPickle as pickle
+import argparse, sys, os.path
+import errno
+import os
+
+
+class DashboardIndex(object):
+
+ def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
+ """
+ :arg host: hostname of the node (default: localhost)
+ :arg port: port to use (integer, default: 9200)
+ :arg url_prefix: optional url prefix for elasticsearch
+ :arg timeout: default timeout in seconds (float, default: 10)
+ """
+ self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
+
+ def get(self):
+ """
+ Get .kibana index from Elasticsearch
+ """
+ dotkibana = self.es.search(index='.kibana', size = 100)
+ return dotkibana['hits']['hits']
+
+ def load(self,filespec):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ data=[]
+ with open(filespec,'rb') as fp:
+ data = pickle.load(fp)
+ return data
+
+ def save(self,filename,data):
+ """
+ Save Index data on local filesystem
+ :args filespec: path/filename for saved file
+ """
+ with open(filename,'wb') as fp:
+ pickle.dump(data,fp)
+
+ def put(self,data):
+ """
+ Bulk write data to Elasticsearch
+ :args data: data to be written (note: index name is specified in data)
+ """
+ bulk(self.es,data)
+
+ def main(self,args):
+
+ if args.save:
+ print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec))
+ self.save(filename=args.filespec,data=di.get())
+ else:
+ """
+ Loads Kibana Dashboard definition from disk and replaces .kibana on index
+ :args filespec: path/filename for saved file
+ """
+ if not os.path.isfile(args.filespec):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
+ self.es.indices.delete(index='.kibana', ignore=[400, 404])
+ self.put(data=di.load(filespec=args.filespec))
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("hostname", help="ES Hostname or IP", type=str)
+ parser.add_argument("port", help="ES Port", type=int)
+ parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
+ parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true")
+ args = parser.parse_args()
+ di = DashboardIndex(host=args.hostname,port=args.port)
+ di.main(args)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
new file mode 100755
index 0000000..10e3a95
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py
@@ -0,0 +1,137 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+kibana_master
+
+"""
+
+import errno
+import os
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+
+class Kibana(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ Logger.info("Install Kibana Master")
+
+ # TODO: Figure this out for all supported OSes
+ Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch')
+ Execute("echo \"[kibana-4.x]\n"
+ "name=Kibana repository for 4.5.x packages\n"
+ "baseurl=http://packages.elastic.co/kibana/4.5/centos\n"
+ "gpgcheck=1\n"
+ "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n"
+ "enabled=1\" > /etc/yum.repos.d/kibana.repo")
+
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+
+ Logger.info("Configure Kibana for Metron")
+
+ directories = [params.log_dir, params.pid_dir, params.conf_dir]
+ Directory(directories,
+ # recursive=True,
+ mode=0755,
+ owner=params.kibana_user,
+ group=params.kibana_user
+ )
+
+ File("{}/kibana.yml".format(params.conf_dir),
+ owner=params.kibana_user,
+ content=InlineTemplate(params.kibana_yml_template)
+ )
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ Logger.info("Stop Kibana Master")
+
+ Execute("service kibana stop")
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+
+ Logger.info("Start the Master")
+
+ Execute("service kibana start")
+
+ def restart(self, env):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+
+ Logger.info("Restarting the Master")
+
+ Execute("service kibana restart")
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+
+ Logger.info("Status of the Master")
+
+ Execute("service kibana status")
+
+ @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+ def load_template(self, env):
+ from dashboard.dashboardindex import DashboardIndex
+
+ import params
+ env.set_params(params)
+
+ hostname = format("{es_host}")
+ port = int(format("{es_port}"))
+
+ Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port))
+ di = DashboardIndex(host=hostname, port=port)
+
+ # Loads Kibana Dashboard definition from disk and replaces .kibana on index
+ templateFile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard.p')
+ if not os.path.isfile(templateFile):
+ raise IOError(
+ errno.ENOENT, os.strerror(errno.ENOENT), templateFile)
+
+ Logger.info("Deleting .kibana index from Elasticsearch")
+
+ di.es.indices.delete(index='.kibana', ignore=[400, 404])
+
+ Logger.info("Loading .kibana index from %s" % templateFile)
+
+ di.put(data=di.load(filespec=templateFile))
+
+
+if __name__ == "__main__":
+ Kibana().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
new file mode 100755
index 0000000..ef1e597
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Kibana Params configurations
+
+"""
+
+from urlparse import urlparse
+
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+kibana_home = '/usr/share/kibana/'
+kibana_bin = '/usr/share/kibana/bin/'
+
+conf_dir = "/opt/kibana/config/"
+kibana_user = config['configurations']['kibana-env']['kibana_user']
+user_group = config['configurations']['kibana-env']['user_group']
+log_dir = config['configurations']['kibana-env']['kibana_log_dir']
+pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
+pid_file = format("{pid_dir}/kibanasearch.pid")
+es_url = config['configurations']['kibana-env']['kibana_es_url']
+parsed = urlparse(es_url)
+es_host = parsed.netloc.split(':')[0]
+es_port = parsed.netloc.split(':')[1]
+kibana_port = config['configurations']['kibana-env']['kibana_server_port']
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+kibana_yml_template = config['configurations']['kibana-site']['content']
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
new file mode 100755
index 0000000..448e102
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"HTTP_ONLY"
+ },
+
+ "links": [
+ {
+ "name": "metron_ui",
+ "label": "Metron UI",
+ "requires_user_name": "false",
+ "component_name": "KIBANA_MASTER",
+ "url":"%@://%@:%@/",
+ "port":{
+ "http_property": "kibana_server_port",
+ "http_default_port": "5601",
+ "https_property": "kibana_server_port",
+ "https_default_port": "5601",
+ "regex": "^(\\d+)$",
+ "site": "kibana-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
new file mode 100644
index 0000000..30f9f76
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/configuration/metron-env.xml
@@ -0,0 +1,174 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_home</name>
+ <value>/usr/metron/0.2.0BETA</value>
+ <description>Metron home directory</description>
+ <display-name>Metron home</display-name>
+ </property>
+ <property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+ </property>
+ <property>
+ <name>metron_zookeeper_config_dir</name>
+ <value>config/zookeeper</value>
+ <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+ <display-name>Metron Zookeeper config dir</display-name>
+ </property>
+ <property>
+ <name>metron_user</name>
+ <value>metron</value>
+ <property-type>USER</property-type>
+ <description>The user for Metron</description>
+ <display-name>Metron User</display-name>
+ </property>
+ <property>
+ <name>metron_group</name>
+ <value>metron</value>
+ <property-type>GROUP</property-type>
+ <description>The group for Metron</description>
+ </property>
+ <property>
+ <name>metron_topic_retention</name>
+ <description>Kafka Retention in GB</description>
+ <value>10</value>
+ </property>
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron parsers</display-name>
+ </property>
+ <property>
+ <name>metron_enrichment_db_user</name>
+ <value>metron</value>
+ <description>Database username to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_port</name>
+ <value>3306</value>
+ <description>Database port to use to connect to the database.</description>
+ </property>
+ <property>
+ <name>metron_enrichment_db_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Metron Enrichment Database Password</display-name>
+ <description>Password to use against database</description>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>metron_indexing_topology</name>
+ <value>indexing</value>
+ <description>The Storm topology name for Indexing</description>
+ </property>
+ <property>
+ <name>es_cluster_name</name>
+ <value>metron</value>
+ <description>Name of Elasticsearch Cluster</description>
+ </property>
+ <property>
+ <name>geoip_url</name>
+ <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
+ <description>Location of the GeoIP data to load.</description>
+ </property>
+ <property require-input="true">
+ <name>es_url</name>
+ <value></value>
+ <description>Comma delimited list of Elasticsearch URLs. (eshost1:9300,eshost2:9300)</description>
+ </property>
+ <property>
+ <name>storm_rest_addr</name>
+ <!--<value-attributes>-->
+ <!--<editable-only-at-install>true</editable-only-at-install>-->
+ <!--<overridable>false</overridable>-->
+ <!--</value-attributes>-->
+ <value></value>
+ </property>
+ <property>
+ <name>global-json</name>
+ <display-name>global.json template</display-name>
+ <description>This is the jinja template for global.json file</description>
+ <value>
+{
+"es.clustername": "{{ es_cluster_name }}",
+"es.ip": "{{ es_url }}",
+"es.date.format": "yyyy.MM.dd.HH"
+}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>elasticsearch-properties</name>
+ <description>The template for the elasticsearch.properties file.</description>
+ <display-name>elasticsearch.properties template</display-name>
+ <value>
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
+##### Kafka #####
+kafka.zk={{ zookeeper_quorum }}
+kafka.broker={{ kafka_brokers }}
+kafka.start=WHERE_I_LEFT_OFF
+##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
+writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
+##### Metrics #####
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+#Graphite Addresses
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+##### HDFS #####
+bolt.hdfs.batch.size=5000
+bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+bolt.hdfs.file.rotation.size.in.mb=5
+bolt.hdfs.file.system.url={{ default_fs }}
+bolt.hdfs.wip.file.path=/paloalto/wip
+bolt.hdfs.finished.file.path=/paloalto/rotated
+bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
+index.hdfs.output=/tmp/metron/enriched
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
[5/7] incubator-metron git commit: METRON-427 Create Ambari
Management Pack for Metron Installation closes apache/incubator-metron#266
Posted by le...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
new file mode 100644
index 0000000..d9886fe
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/metainfo.xml
@@ -0,0 +1,273 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>METRON</name>
+ <displayName>Metron</displayName>
+ <comment>A scalable advanced security analytics framework built on Hadoop</comment>
+ <version>0.2.0BETA</version>
+ <components>
+
+ <component>
+ <name>METRON_PARSERS</name>
+ <displayName>Metron Parsers</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/parser_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>metron-parsers</config-type>
+ </configuration-dependencies>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MASTER</name>
+ <displayName>Metron Enrichment</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/enrichment_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
+ <displayName>MySQL Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>CLEAN</name>
+ <commandScript>
+ <script>scripts/mysql_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
+ <component>
+ <name>METRON_INDEXING</name>
+ <displayName>Metron Indexing</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_INSTALL</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>ELASTICSEARCH_TEMPLATE_DELETE</name>
+ <commandScript>
+ <script>scripts/indexing_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ <configuration-dependencies>
+ <config-type>metron-indexing</config-type>
+ </configuration-dependencies>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>metron-common</name>
+ </package>
+ <package>
+ <name>metron-parsers</name>
+ </package>
+ <package>
+ <name>metron-enrichment</name>
+ </package>
+ <package>
+ <name>metron-indexing</name>
+ </package>
+ <package>
+ <name>metron-elasticsearch</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>mysql-community-release</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql-community-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>MySQL-python</package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat6</osFamily>
+ <packages>
+ <package>
+ <name>mysql-server</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>mysql</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ <service>KAFKA</service>
+ <service>STORM</service>
+ <service>ZOOKEEPER</service>
+ <service>HBASE</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>metron-env</config-type>
+ <config-type>storm-env</config-type>
+ <config-type>storm-site</config-type>
+ <config-type>kafka-broker</config-type>
+ <config-type>kafka-env</config-type>
+ </configuration-dependencies>
+ <restartRequiredAfterChange>true</restartRequiredAfterChange>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
new file mode 100755
index 0000000..5dafcb8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/addMysqlUser.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+mysqldbuser=$2
+mysqldbpasswd=$3
+mysqldbhost=$4
+myhostname=$(hostname -f)
+
+service $mysqldservice start
+echo "Adding user $mysqldbuser@$mysqldbhost and $mysqldbuser@localhost"
+mysql -u root -e "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';"
+mysql -u root -e "CREATE USER '$mysqldbuser'@'localhost' IDENTIFIED BY '$mysqldbpasswd';"
+
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'localhost';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'%' IDENTIFIED BY '$mysqldbpasswd';"
+
+if [ '$(mysql -u root -e "select user from mysql.user where user='$mysqldbuser' and host='$myhostname'" | grep "$mysqldbuser")' != '0' ]; then
+ echo "Adding user $mysqldbuser@$myhostname";
+ mysql -u root -e "CREATE USER '$mysqldbuser'@'$myhostname' IDENTIFIED BY '$mysqldbpasswd';";
+ mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$myhostname';";
+fi
+mysql -u root -e "flush privileges;"
+service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
new file mode 100644
index 0000000..951d967
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/bro_index.template
@@ -0,0 +1,217 @@
+{
+ "template": "bro_index*",
+ "mappings": {
+ "bro_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "status_code": {
+ "type": "integer"
+ },
+ "method": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "request_body_len": {
+ "type": "integer"
+ },
+ "uri": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "referrer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_depth": {
+ "type": "integer"
+ },
+ "host": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "response_body_len": {
+ "type": "integer"
+ },
+ "user_agent": {
+ "type": "string"
+ },
+ "query": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "answers": {
+ "type": "ip"
+ },
+ "AA": {
+ "type": "boolean"
+ },
+ "TC": {
+ "type": "boolean"
+ },
+ "RA": {
+ "type": "boolean"
+ },
+ "RD": {
+ "type": "boolean"
+ },
+ "rejected": {
+ "type": "boolean"
+ },
+ "qclass_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rcode": {
+ "type": "integer"
+ },
+ "rcode_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_id": {
+ "type": "integer"
+ },
+ "Z": {
+ "type": "integer"
+ },
+ "qclass": {
+ "type": "integer"
+ },
+ "qtype": {
+ "type": "integer"
+ },
+ "qtype_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
new file mode 100755
index 0000000..21ced42
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/createMysqlGeoIp.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+geoipscript=$2
+geoipurl=$3
+
+# Download and extract the actual GeoIP files
+mkdir -p /tmp/geoip
+mkdir -p /var/lib/mysql-files/
+
+# Stage the GeoIP data
+pushd /tmp/geoip
+curl -O ${geoipurl}
+tar xf GeoLiteCity-latest.tar.xz
+cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
+popd
+
+# Load MySQL with the GeoIP data and start service
+service ${mysqldservice} start
+mysql -u root < ${geoipscript}
+mysql -u root -e "show databases;"
+service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
new file mode 100755
index 0000000..33a5ee5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/removeMysqlUser.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldservice=$1
+mysqldbuser=$2
+userhost=$3
+myhostname=$(hostname -f)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+
+${sudo_prefix} service ${mysqldservice} start
+echo "Removing user $mysqldbuser@$userhost"
+/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"DROP USER '$mysqldbuser'@'$userhost';\""
+/var/lib/ambari-agent/ambari-sudo.sh su mysql -s /bin/bash - -c "mysql -u root -e \"flush privileges;\""
+${sudo_prefix} service ${mysqldservice} stop
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
new file mode 100644
index 0000000..bf943df
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/snort_index.template
@@ -0,0 +1,183 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "dgmlen": {
+ "type": "integer"
+ },
+ "ethdst": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethlen": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ethsrc": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "iplen": {
+ "type": "integer"
+ },
+ "is_alert": {
+ "type": "boolean"
+ },
+ "msg": {
+ "type": "string"
+ },
+ "protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_generator": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sig_id": {
+ "type": "integer"
+ },
+ "sig_rev": {
+ "type": "string"
+ },
+ "tcpack": {
+ "type": "string"
+ },
+ "tcpflags": {
+ "type": "string"
+ },
+ "tcpseq": {
+ "type": "string"
+ },
+ "tcpwindow": {
+ "type": "string"
+ },
+ "threat:triage:level": {
+ "type": "double"
+ },
+ "tos": {
+ "type": "integer"
+ },
+ "ttl": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
new file mode 100644
index 0000000..7743afc
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/files/yaf_index.template
@@ -0,0 +1,205 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "start_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "end_time": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "duration": {
+ "type": "double"
+ },
+ "rtt": {
+ "type": "double"
+ },
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "dp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "iflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "isn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "risn": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rtag": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "pkt": {
+ "type": "integer"
+ },
+ "oct": {
+ "type": "integer"
+ },
+ "rpkt": {
+ "type": "integer"
+ },
+ "roct": {
+ "type": "integer"
+ },
+ "app": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
new file mode 100755
index 0000000..027ffae
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_commands.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class EnrichmentCommands:
+ __params = None
+ __enrichment_topology = None
+ __enrichment_topic = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__enrichment_topology = params.metron_enrichment_topology
+ self.__enrichment_topic = params.metron_enrichment_topic
+ self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.enrichment_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for enrichment")
+
+ Logger.info("Creating topic'{0}'".format(self.__enrichment_topic))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__enrichment_topic,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_enrichment_topology(self):
+ Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
+ start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__enrichment_topology)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__enrichment_topology, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting enrichment topology')
+
+ def stop_enrichment_topology(self):
+ Logger.info('Stopping ' + self.__enrichment_topology)
+ stop_cmd = 'storm kill ' + self.__enrichment_topology
+ Execute(stop_cmd)
+ Logger.info('Done stopping enrichment topologies')
+
+ def restart_enrichment_topology(self, env):
+ Logger.info('Restarting the enrichment topologies')
+ self.stop_enrichment_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while topology_active and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(40)
+ topology_active = self.is_topology_active(env)
+ retries += 1
+
+ if not topology_active:
+ self.start_enrichment_topology()
+ Logger.info('Done restarting the enrichment topology')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__enrichment_topology in topologies:
+ is_running = topologies[self.__enrichment_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def create_hbase_tables(self):
+ add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ Execute(add_enrichment_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+ add_threatintel_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ Execute(add_threatintel_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
new file mode 100755
index 0000000..390576b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/enrichment_master.py
@@ -0,0 +1,87 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+
+from enrichment_commands import EnrichmentCommands
+
+
+class Enrichment(Script):
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ File(format("{metron_config_path}/enrichment.properties"),
+ content=Template("enrichment.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.create_hbase_tables()
+ commands.set_configured()
+
+ commands.start_enrichment_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.stop_enrichment_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = EnrichmentCommands(status_params)
+
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.restart_enrichment_topology(env)
+
+ def kafkabuild(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = EnrichmentCommands(params)
+ commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+ Enrichment().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
new file mode 100755
index 0000000..8a7e3c6
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_commands.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+
+
+# Wrap major operations and functionality in this class
+class IndexingCommands:
+ __params = None
+ __indexing = None
+ __configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__indexing = params.metron_indexing_topology
+ self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
+
+ def is_configured(self):
+ return self.__configured
+
+ def set_configured(self):
+ File(self.__params.indexing_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0775)
+
+ def setup_repo(self):
+ def local_repo():
+ Logger.info("Setting up local repo")
+ Execute("yum -y install createrepo")
+ Execute("createrepo /localrepo")
+ Execute("chmod -R o-w+r /localrepo")
+ Execute("echo \"[METRON-0.2.0BETA]\n"
+ "name=Metron 0.2.0BETA packages\n"
+ "baseurl=file:///localrepo\n"
+ "gpgcheck=0\n"
+ "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+ def remote_repo():
+ print('Using remote repo')
+
+ yum_repo_types = {
+ 'local': local_repo,
+ 'remote': remote_repo
+ }
+ repo_type = self.__params.yum_repo_type
+ if repo_type in yum_repo_types:
+ yum_repo_types[repo_type]()
+ else:
+ raise ValueError("Unsupported repo type '{0}'".format(repo_type))
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics')
+ command_template = """{0}/kafka-topics.sh \
+ --zookeeper {1} \
+ --create \
+ --topic {2} \
+ --partitions {3} \
+ --replication-factor {4} \
+ --config retention.bytes={5}"""
+ num_partitions = 1
+ replication_factor = 1
+ retention_gigabytes = int(self.__params.metron_topic_retention)
+ retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+ Logger.info("Creating topics for indexing")
+
+ Logger.info("Creating topic'{0}'".format(self.__indexing))
+ Execute(command_template.format(self.__params.kafka_bin_dir,
+ self.__params.zookeeper_quorum,
+ self.__indexing,
+ num_partitions,
+ replication_factor,
+ retention_bytes))
+ Logger.info("Done creating Kafka topics")
+
+ def start_indexing_topology(self):
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
+ -s {1} \
+ -z {2}"""
+ Logger.info('Starting ' + self.__indexing)
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum))
+
+ Logger.info('Finished starting indexing topology')
+
+ def stop_indexing_topology(self):
+ Logger.info('Stopping ' + self.__indexing)
+ stop_cmd = 'storm kill ' + self.__indexing
+ Execute(stop_cmd)
+ Logger.info('Done stopping indexing topologies')
+
+ def restart_indexing_topology(self, env):
+ Logger.info('Restarting the indexing topologies')
+ self.stop_indexing_topology()
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_indexing_topology()
+ Logger.info('Done restarting the indexing topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies()
+ is_running = False
+ if self.__indexing in topologies:
+ is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
new file mode 100755
index 0000000..eabf5a1
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/indexing_master.py
@@ -0,0 +1,120 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+import metron_service
+from indexing_commands import IndexingCommands
+
+
+class Indexing(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.setup_repo()
+ Logger.info('Install RPM packages')
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ commands = IndexingCommands(params)
+ metron_service.load_global_config(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.start_indexing_topology()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = IndexingCommands(params)
+ commands.stop_indexing_topology()
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = IndexingCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = IndexingCommands(params)
+ commands.restart_indexing_topology(env)
+
+ def elasticsearch_template_install(self, env):
+ from params import params
+ env.set_params(params)
+
+ File(params.bro_index_path,
+ mode=0755,
+ content=StaticFile('bro_index.template')
+ )
+
+ File(params.snort_index_path,
+ mode=0755,
+ content=StaticFile('snort_index.template')
+ )
+
+ File(params.yaf_index_path,
+ mode=0755,
+ content=StaticFile('yaf_index.template')
+ )
+
+ bro_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/bro_index -d @{bro_index_path}')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/snort_index -d @{snort_index_path}')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_url}/_template/yaf_index -d @{yaf_index_path}')
+ Execute(yaf_cmd, logoutput=True)
+
+ def elasticsearch_template_delete(self, env):
+ from params import params
+ env.set_params(params)
+
+ bro_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/bro_index*"')
+ Execute(bro_cmd, logoutput=True)
+ snort_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/snort_index*"')
+ Execute(snort_cmd, logoutput=True)
+ yaf_cmd = ambari_format('curl -s -XDELETE "http://{es_url}/yaf_index*"')
+ Execute(yaf_cmd, logoutput=True)
+
+
+if __name__ == "__main__":
+ Indexing().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
new file mode 100644
index 0000000..57da2c7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/metron_service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format as ambari_format
+
+
+def init_config():
+ Logger.info('Loading config into ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+
+def get_running_topologies():
+ Logger.info('Getting Running Storm Topologies from Storm REST Server')
+
+ cmd = ambari_format('curl --max-time 3 {storm_rest_addr}/api/v1/topology/summary')
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = proc.communicate()
+
+ try:
+ stormjson = json.loads(stdout)
+ except ValueError:
+ return {}
+
+ topologiesDict = {}
+
+ for topology in stormjson['topologies']:
+ topologiesDict[topology['name']] = topology['status']
+
+ Logger.info("Topologies: " + str(topologiesDict))
+ return topologiesDict
+
+
+def load_global_config(params):
+ Logger.info('Create Metron Local Config Directory')
+ Logger.info("Configure Metron global.json")
+
+ directories = [params.metron_zookeeper_config_path]
+ Directory(directories,
+ mode=0755,
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
+ File("{0}/global.json".format(params.metron_zookeeper_config_path),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_json_template)
+ )
+
+ File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'),
+ owner=params.metron_user,
+ content=InlineTemplate(params.global_properties_template))
+
+ init_config()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
new file mode 100755
index 0000000..2fc363f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_server.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.script import Script
+
+import mysql_users
+from mysql_service import mysql_service
+from mysql_utils import mysql_configure
+
+
+class MysqlServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def clean(self, env):
+ from params import params
+ env.set_params(params)
+ mysql_users.mysql_deluser()
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+ mysql_configure()
+
+ def start(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='start')
+
+ def stop(self, env, rolling_restart=False):
+ from params import params
+ env.set_params(params)
+ mysql_service(daemon_name=params.daemon_name, action='stop')
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+
+ mysql_service(daemon_name=status_params.daemon_name, action='status')
+
+
+if __name__ == "__main__":
+ MysqlServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
new file mode 100755
index 0000000..2e0ce8b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning, Fail
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+
+
+def mysql_service(daemon_name=None, action='start'):
+ status_cmd = format("pgrep -l '^{mysql_process_name}$'")
+ cmd = ('service', daemon_name, action)
+
+ if action == 'status':
+ try:
+ Execute(status_cmd)
+ except Fail:
+ raise ComponentIsNotRunning()
+ elif action == 'stop':
+ Execute(cmd,
+ logoutput=True,
+ only_if=status_cmd,
+ sudo=True,
+ )
+ elif action == 'start':
+ Execute(cmd,
+ logoutput=True,
+ not_if=status_cmd,
+ sudo=True,
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
new file mode 100755
index 0000000..1721eba
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_users.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+
+# Used to add metron access to the needed components
+def mysql_adduser():
+ from params import params
+
+ File(params.mysql_adduser_path,
+ mode=0755,
+ content=StaticFile('addMysqlUser.sh')
+ )
+
+ add_user_cmd = format("bash -x {mysql_adduser_path} {daemon_name} {metron_user} {enrichment_metron_user_passwd!p} {enrichment_host}")
+ Execute(add_user_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ )
+
+
+# Removes hive metron from components
+def mysql_deluser():
+ from params import params
+
+ File(params.mysql_deluser_path,
+ mode=0755,
+ content=StaticFile('removeMysqlUser.sh')
+ )
+
+ del_user_cmd = format("bash -x {mysql_deluser_path} {daemon_name} {metron_user} {enrichment_host}")
+ Execute(del_user_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
new file mode 100755
index 0000000..40b925d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/mysql_utils.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+
+import mysql_users
+
+
+def mysql_configure():
+ from params import params
+
+ # required for running hive
+ replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
+ Execute(replace_bind_address,
+ sudo=True,
+ )
+
+ # this also will start mysql-server
+ mysql_users.mysql_adduser()
+
+ File(params.mysql_create_geoip_path,
+ mode=0755,
+ content=StaticFile('createMysqlGeoIp.sh')
+ )
+
+ geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {daemon_name} {geoip_ddl} {geoip_url}")
+ Execute(geoip_setup_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ )
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..5f914f3
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_linux.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hostname = config['hostname']
+metron_home = status_params.metron_home
+parsers = status_params.parsers
+metron_ddl_dir = metron_home + '/ddl'
+geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
+geoip_url = config['configurations']['metron-env']['geoip_url']
+metron_indexing_topology = status_params.metron_indexing_topology
+metron_user = config['configurations']['metron-env']['metron_user']
+metron_group = config['configurations']['metron-env']['metron_group']
+metron_config_path = metron_home + '/config'
+metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
+metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+parsers_configured_flag_file = status_params.parsers_configured_flag_file
+enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+indexing_configured_flag_file = status_params.indexing_configured_flag_file
+global_json_template = config['configurations']['metron-env']['global-json']
+global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
+es_cluster_name = config['configurations']['metron-env']['es_cluster_name']
+es_url = config['configurations']['metron-env']['es_url']
+yum_repo_type = 'local'
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Storm
+storm_rest_addr = status_params.storm_rest_addr
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+ if 'port' in config['configurations']['kafka-broker']:
+ kafka_broker_port = config['configurations']['kafka-broker']['port']
+ else:
+ kafka_broker_port = '6667'
+ kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+ kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# MYSQL
+if OSCheck.is_ubuntu_family():
+ mysql_configname = '/etc/mysql/my.cnf'
+else:
+ mysql_configname = '/etc/my.cnf'
+
+daemon_name = status_params.daemon_name
+# There will always be exactly one mysql_host
+mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
+mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
+
+mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
+mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
+mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
+
+enrichment_hosts = default("/clusterHostInfo/enrichment_host", [])
+enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None
+
+enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
+enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
+enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
+ enrichment_metron_user_passwd) else enrichment_metron_user_passwd
+mysql_process_name = status_params.mysql_process_name
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ hadoop_bin_dir=hadoop_bin_dir,
+ hadoop_conf_dir=hadoop_conf_dir,
+ principal_name=hdfs_principal_name,
+ hdfs_site=hdfs_site,
+ default_fs=default_fs,
+ immutable_paths=get_not_managed_resources(),
+ dfs_type=dfs_type
+)
+
+# HBase
+enrichment_table = status_params.enrichment_table
+enrichment_cf = status_params.enrichment_cf
+threatintel_table = status_params.threatintel_table
+threatintel_cf = status_params.threatintel_cf
+
+metron_enrichment_topology = status_params.metron_enrichment_topology
+metron_enrichment_topic = status_params.metron_enrichment_topic
+
+# ES Templates
+bro_index_path = tmp_dir + "/bro_index.template"
+snort_index_path = tmp_dir + "/snort_index.template"
+yaf_index_path = tmp_dir + "/yaf_index.template"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
new file mode 100644
index 0000000..a0310d2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/0.2.0BETA/package/scripts/params/status_params.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons import OSCheck
+from resource_management.libraries.functions import format
+from resource_management.libraries.script import Script
+
+config = Script.get_config()
+
+# Parsers
+parsers = config['configurations']['metron-env']['parsers']
+metron_home = config['configurations']['metron-env']['metron_home']
+metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
+
+# Enrichment
+metron_enrichment_topology = 'enrichment'
+metron_enrichment_topic = 'enrichments'
+
+enrichment_table = 'enrichment'
+enrichment_cf = 't'
+threatintel_table = 'threatintel'
+threatintel_cf = 't'
+
+mysql_process_name = 'mysqld'
+if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
+ daemon_name = 'mysql'
+else:
+ daemon_name = 'mysqld'
+
+# ing
+metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
+indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
+
+# Enrichment
+enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_is_configured'
+
+# Storm
+storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']