You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@metron.apache.org by "Neha Sinha (JIRA)" <ji...@apache.org> on 2016/10/31 07:27:58 UTC
[jira] [Created] (METRON-525) Unable to start PCAP topology
Neha Sinha created METRON-525:
---------------------------------
Summary: Unable to start PCAP topology
Key: METRON-525
URL: https://issues.apache.org/jira/browse/METRON-525
Project: Metron
Issue Type: Bug
Affects Versions: 0.2.2BETA
Reporter: Neha Sinha
The following error is seen while starting PCAP topology :-
=========================================================
[root@metron-s-10 ~]# /usr/metron/0.2.1BETA/bin/start_parser_topology.sh -k metron-s-10.openstacklocal:6667 -z metron-s-10.openstacklocal:2181 -s pcap
Running: /usr/jdk64/jdk1.8.0_77/bin/java -client -Ddaemon.name= -Dstorm.options= -Dstorm.home=/grid/0/hdp/2.4.3.0-227/storm -Dstorm.log.dir=/grid/0/log/storm -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdp/current/storm-client/lib -Dstorm.conf.file= -cp /grid/0/hdp/2.4.3.0-227/storm/lib/log4j-api-2.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/cheshire-5.3.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/compojure-1.1.3.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/tools.logging-0.2.3.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/core.incubator-0.1.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/jline-0.9.94.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ring-core-1.1.5.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/java.classpath-0.2.2.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/slf4j-api-1.7.7.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/zookeeper.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/disruptor-2.10.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/log4j-core-2.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/jackson-core-2.3.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/tigris-0.1.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/reflectasm-1.07-shaded.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/clj-stacktrace-0.2.7.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/commons-codec-1.6.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/clojure-1.6.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ring-jetty-adapter-1.3.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ring-json-0.3.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/servlet-api-2.5.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/tools.namespace-0.2.4.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/clj-time-0.8.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ring-devel-1.3.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/hadoop-auth-2.7.1.2.4.3.0-227.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/jackson-dataformat-smile-2.3.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/hiccup-0.3.6.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/asm-4.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/storm-core-0.10.0.2.4.3.0-227.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/clout-1.0.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ns-tracker-0.2.2.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/minlog-1.2.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/oncrpc-1.0.7.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/log4j-slf4j-impl-2.1.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/gmetric4j-1.0.7.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/ring-servlet-1.3.0.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/javax.servlet-2.5.0.v201103041518.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/kryo-2.21.jar:/grid/0/hdp/2.4.3.0-227/storm/lib/log4j-over-slf4j-1.6.6.jar:/usr/metron/0.2.1BETA/lib/metron-parsers-0.2.1BETA-uber.jar:/usr/hdp/current/storm-supervisor/conf:/grid/0/hdp/2.4.3.0-227/storm/bin -Dstorm.jar=/usr/metron/0.2.1BETA/lib/metron-parsers-0.2.1BETA-uber.jar org.apache.metron.parsers.topology.ParserTopologyCLI -k metron-s-10.openstacklocal:6667 -z metron-s-10.openstacklocal:2181 -s pcap
05:59:01.065 [main] INFO o.a.c.f.i.CuratorFrameworkImpl - Starting
05:59:01.156 [main-EventThread] INFO o.a.c.f.s.ConnectionStateManager - State change: CONNECTED
java.lang.IllegalStateException: Cannot find the parser configuration in zookeeper for pcap. Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.
at org.apache.metron.parsers.topology.ParserTopologyBuilder.getSensorParserConfig(ParserTopologyBuilder.java:225)
at org.apache.metron.parsers.topology.ParserTopologyBuilder.build(ParserTopologyBuilder.java:85)
at org.apache.metron.parsers.topology.ParserTopologyCLI.main(ParserTopologyCLI.java:298)
=========================================================
zk_load_configs.sh -m DUMP output
========================================================
[root@metron-s-10 ~]# /usr/metron/0.2.1BETA/bin/zk_load_configs.sh -m DUMP -z metron-s-10.openstacklocal:2181
log4j:WARN No appenders could be found for logger (org.apache.curator.framework.imps.CuratorFrameworkImpl).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
GLOBAL Config: global
{
"es.clustername": "metron",
"es.ip": "metron-s-10.openstacklocal:9300",
"es.date.format": "yyyy.MM.dd.HH"
}
PARSER Config: websphere
{
"parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser",
"sensorTopic":"websphere",
"parserConfig":
{
"grokPath":"/patterns/websphere",
"patternLabel":"WEBSPHERE",
"timestampField":"timestamp_string",
"dateFormat":"yyyy MMM dd HH:mm:ss"
}
}
PARSER Config: squid
{
"parserClassName": "org.apache.metron.parsers.GrokParser",
"sensorTopic": "squid",
"parserConfig": {
"grokPath": "/patterns/squid",
"patternLabel": "SQUID_DELIMITED",
"timestampField": "timestamp"
},
"fieldTransformations" : [
{
"transformation" : "STELLAR"
,"output" : [ "full_hostname", "domain_without_subdomains" ]
,"config" : {
"full_hostname" : "URL_TO_HOST(url)"
,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
}
}
]
}
PARSER Config: jsonMap
{
"parserClassName":"org.apache.metron.parsers.json.JSONMapParser",
"sensorTopic":"jsonMap"
}
PARSER Config: bro
{
"parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
"sensorTopic":"bro",
"parserConfig": {}
}
PARSER Config: snort
{
"parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
"sensorTopic":"snort",
"parserConfig": {}
}
PARSER Config: yaf
{
"parserClassName":"org.apache.metron.parsers.GrokParser",
"sensorTopic":"yaf",
"fieldTransformations" : [
{
"input" : "protocol"
,"transformation": "IP_PROTOCOL"
}
],
"parserConfig":
{
"grokPath":"/patterns/yaf",
"patternLabel":"YAF_DELIMITED",
"timestampField":"start_time",
"timeFields": ["start_time", "end_time"],
"dateFormat":"yyyy-MM-dd HH:mm:ss.S"
}
}
ENRICHMENT Config: websphere
{
"index": "websphere",
"batchSize": 5,
"enrichment": {
"fieldMap": {
"geo": [
"ip_src_addr"
],
"host": [
"ip_src_addr"
]
},
"fieldToTypeMap": {
"ip_src_addr": [
"playful_classification"
]
}
}
}
ENRICHMENT Config: bro
{
"index": "bro",
"batchSize": 5,
"enrichment" : {
"fieldMap": {
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel": {
"fieldMap": {
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap": {
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
}
}
}
ENRICHMENT Config: snort
{
"index": "snort",
"batchSize": 1,
"enrichment" : {
"fieldMap":
{
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel" : {
"fieldMap":
{
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap":
{
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
},
"triageConfig" : {
"riskLevelRules" : {
"not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))" : 10
},
"aggregator" : "MAX"
}
}
}
ENRICHMENT Config: yaf
{
"index": "yaf",
"batchSize": 5,
"enrichment" : {
"fieldMap":
{
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel": {
"fieldMap":
{
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap":
{
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
}
}
}
[root@metron-s-10 ~]#
========================================================
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)