You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@metron.apache.org by mmiklavc <gi...@git.apache.org> on 2018/06/05 20:09:30 UTC

[GitHub] metron issue #1051: METRON-1603: Fix multivalue field errors in Bro Solr sch...

Github user mmiklavc commented on the issue:

    https://github.com/apache/metron/pull/1051
  
    **Testing**
    
    I have proper unit tests, but you can validate the change by manually piping data into Metron.
    
    1. Make sure you're running Solr
    2. Put the following in a sample data file, e.g. test-bro.txt:
        
    ```
    {"adapter.threatinteladapter.end.ts":"1528192727455","bro_timestamp":"1402307733.473","enrichments.geo.ip_src_addr.longitude":"-118.4041","enrichmentsplitterbolt.splitter.end.ts":"1528192727437","enrichments.geo.ip_dst_addr.city":"Richardson","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"4722625","enrichments.geo.ip_src_addr.city":"Los Angeles","resp_mime_types":["text\/html","text\/xml"],"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[\"a\",\"b\",\"c\"] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\",\"text\\\/xml\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\",\"GJDyMC15lxUn5ngPfe\"]","enrichments.geo.ip_dst_addr.
 dmaCode":"623","host":"www.cisco.com","enrichmentjoinbolt.joiner.ts":"1528192727444","adapter.hostfromjsonlistadapter.begin.ts":"1528192727439","enrichments.geo.ip_src_addr.dmaCode":"803","method":"GET","tags":["a","b","c"],"adapter.geoadapter.end.ts":"1528192727442","adapter.threatinteladapter.begin.ts":"1528192727455","enrichments.geo.ip_dst_addr.location_point":"32.9513,-96.7154","guid":"68731e82-6a23-4d5c-97f4-9701490a99dc","response_body_len":25523,"status_code":200,"ip_dst_port":80,"enrichments.geo.ip_src_addr.location_point":"33.9571,-118.4041","enrichments.geo.ip_dst_addr.latitude":"32.9513","enrichmentsplitterbolt.splitter.begin.ts":"1528192727437","adapter.hostfromjsonlistadapter.end.ts":"1528192727439","adapter.geoadapter.begin.ts":"1528192727442","enrichments.geo.ip_dst_addr.postalCode":"75081","enrichments.geo.ip_src_addr.postalCode":"90045","uid":"CTo78A11g7CYbbOHvj","trans_depth":1,"ip_dst_addr":"72.163.4.161","enrichments.geo.ip_src_addr.latitude":"33.9571","threatin
 teljoinbolt.joiner.ts":"1528192727458","threatintelsplitterbolt.splitter.begin.ts":"1528192727446","enrichments.geo.ip_src_addr.locID":"5368361","enrichments.geo.ip_dst_addr.longitude":"-96.7154","ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","enrichments.geo.ip_src_addr.country":"US","resp_fuids":["FJDyMC15lxUn5ngPfd","GJDyMC15lxUn5ngPfe"],"timestamp":1402307733473,"request_body_len":0,"uri":"\/","source.type":"bro","threatintelsplitterbolt.splitter.end.ts":"1528192727446","ip_src_port":58808,"status_msg":"OK"}
    {"TTLs":[3600.0,289.0,14.0],"adapter.threatinteladapter.end.ts":"1528192727455","qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"AAAA","ip_dst_port":53,"enrichmentsplitterbolt.splitter.end.ts":"1528192727437","qtype":28,"rejected":false,"enrichments.geo.ip_dst_addr.city":"Almere Stad","enrichments.geo.ip_dst_addr.latitude":"52.3881","answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"enrichmentsplitterbolt.splitter.begin.ts":"1528192727437","adapter.hostfromjsonlistadapter.end.ts":"1528192727439","enrichments.geo.ip_dst_addr.country":"NL","enrichments.geo.ip_dst_addr.locID":"2759879","trans_id":62418,"adapter.geoadapter.begin.ts":"1528192727442","enrichments.geo.ip_dst_addr.postalCode":"1317","uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 qtype_name:AAAA qtype:28 rejected:false id.resp_p:53 query:www.cisco.com answers:[\"www.cisco.
 com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"] trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 ts:1402308259.609 id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","threatinteljoinbolt.joiner.ts":"1528192727458","enrichmentjoinbolt.joiner.ts":"1528192727445","adapter.hostfromjsonlistadapter.begin.ts":"1528192727439","threatintelsplitterbolt.splitter.begin.ts":"1528192727446","Z":0,"enrichments.geo.ip_dst_addr.longitude":"5.2354","ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","adapter.geoadapter.end.ts":"1528192727442","RD":true,"threatintelsplitterbolt.splitter.end.ts":"1528192727446","adapter.threatinteladapter.begin.ts":"1528192727455","ip_src_port":33976,"proto":"udp","enrichments.geo.ip_dst_addr.location_point":"52.3881,5.2354","guid":"d32
 0cb1c-e4dc-4b1d-9650-75bcf2c9e371"}
    {"adapter.threatinteladapter.end.ts":"1528192727455","bro_timestamp":"1216706983.387664","timedout":true,"enrichments.geo.ip_src_addr.longitude":"-118.244","enrichmentsplitterbolt.splitter.end.ts":"1528192727438","enrichments.geo.ip_src_addr.location_point":"34.0544,-118.244","enrichmentsplitterbolt.splitter.begin.ts":"1528192727438","adapter.hostfromjsonlistadapter.end.ts":"1528192727440","source":"HTTP","adapter.geoadapter.begin.ts":"1528192727442","duration":30.701792,"protocol":"files","original_string":"FILES | timedout:true rx_hosts:[\"192.168.15.4\",\"192.168.15.5\"] source:HTTP is_orig:false tx_hosts:[\"216.113.185.92\",\"216.113.185.93\"] overflow_bytes:0 duration:30.701792 depth:0 analyzers:[\"MD5\",\"SHA1\"] fuid:FnEYba9VPOcC41c1 conn_uids:[\"CLWqoN1IA9MB8Ru9i3\",\"DLWqoN1IA9MB8Ru9i4\"] seen_bytes:0 missing_bytes:3384 ts:1216706983.387664","ip_dst_addr":"192.168.15.4","analyzers":["MD5","SHA1"],"enrichments.geo.ip_src_addr.latitude":"34.0544","threatinteljoinbolt.joine
 r.ts":"1528192727458","enrichmentjoinbolt.joiner.ts":"1528192727445","adapter.hostfromjsonlistadapter.begin.ts":"1528192727440","threatintelsplitterbolt.splitter.begin.ts":"1528192727446","fuid":"FnEYba9VPOcC41c1","seen_bytes":0,"missing_bytes":3384,"ip_src_addr":"216.113.185.92","enrichments.geo.ip_src_addr.country":"US","timestamp":1216706983387,"is_orig":false,"overflow_bytes":0,"source.type":"bro","adapter.geoadapter.end.ts":"1528192727442","depth":0,"threatintelsplitterbolt.splitter.end.ts":"1528192727446","adapter.threatinteladapter.begin.ts":"1528192727455","guid":"558bb655-3867-439b-b26d-13aa77d1b3ec","conn_uids":["CLWqoN1IA9MB8Ru9i3","DLWqoN1IA9MB8Ru9i4"]}
    {"adapter.threatinteladapter.end.ts":"1528192727455","bro_timestamp":"1440447880.931272","resp_pkts":1,"ip_dst_port":1812,"enrichmentsplitterbolt.splitter.end.ts":"1528192727439","enrichmentsplitterbolt.splitter.begin.ts":"1528192727439","adapter.hostfromjsonlistadapter.end.ts":"1528192727441","adapter.geoadapter.begin.ts":"1528192727442","duration":1.001459,"uid":"CWxtRHnBTbldHnmGh","protocol":"conn","original_string":"CONN | id.orig_p:52178 resp_pkts:1 resp_ip_bytes:48 orig_bytes:75 id.resp_p:1812 orig_ip_bytes:103 orig_pkts:1 missed_bytes:0 history:Dd tunnel_parents:[\"a\",\"b\",\"c\"] duration:1.001459 uid:CWxtRHnBTbldHnmGh resp_bytes:20 service:radius conn_state:SF proto:udp id.orig_h:127.0.0.1 ts:1440447880.931272 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","threatinteljoinbolt.joiner.ts":"1528192727458","conn_state":"SF","enrichmentjoinbolt.joiner.ts":"1528192727445","adapter.hostfromjsonlistadapter.begin.ts":"1528192727441","threatintelsplitterbolt.splitter.begin.ts":"
 1528192727446","ip_src_addr":"127.0.0.1","timestamp":1440447880931,"resp_ip_bytes":48,"orig_bytes":75,"orig_ip_bytes":103,"orig_pkts":1,"missed_bytes":0,"history":"Dd","tunnel_parents":["a","b","c"],"source.type":"bro","adapter.geoadapter.end.ts":"1528192727442","resp_bytes":20,"threatintelsplitterbolt.splitter.end.ts":"1528192727446","adapter.threatinteladapter.begin.ts":"1528192727455","ip_src_port":52178,"service":"radius","proto":"udp","guid":"d599c0a8-46f5-44d5-a504-409790d7468a"}
    {"adapter.threatinteladapter.end.ts":"1528192727458","bro_timestamp":"1258568036.57884","ip_dst_port":25,"enrichmentsplitterbolt.splitter.end.ts":"1528192727442","enrichmentsplitterbolt.splitter.begin.ts":"1528192727441","adapter.hostfromjsonlistadapter.end.ts":"1528192727444","adapter.geoadapter.begin.ts":"1528192727444","uid":"ChR6254RrWbrxiGsd7","path":["192.168.1.1","192.168.1.105"],"trans_depth":1,"protocol":"smtp","original_string":"SMTP | id.orig_p:49353 id.resp_p:25 helo:M57Terry uid:ChR6254RrWbrxiGsd7 path:[\"192.168.1.1\",\"192.168.1.105\"] trans_depth:1 is_webmail:false last_reply:220 2.0.0 Ready to start TLS id.orig_h:192.168.1.105 tls:true fuids:[\"a\",\"b\",\"c\"] ts:1258568036.57884 id.resp_h:192.168.1.1","ip_dst_addr":"192.168.1.1","is_webmail":false,"threatinteljoinbolt.joiner.ts":"1528192727460","enrichmentjoinbolt.joiner.ts":"1528192727447","adapter.hostfromjsonlistadapter.begin.ts":"1528192727444","threatintelsplitterbolt.splitter.begin.ts":"1528192727455","fu
 ids":["a","b","c"],"ip_src_addr":"192.168.1.105","timestamp":1258568036578,"source.type":"bro","helo":"M57Terry","adapter.geoadapter.end.ts":"1528192727444","threatintelsplitterbolt.splitter.end.ts":"1528192727455","adapter.threatinteladapter.begin.ts":"1528192727457","ip_src_port":49353,"last_reply":"220 2.0.0 Ready to start TLS","guid":"c1ca10a2-615b-4038-be57-5c9790743477","tls":true}
    {"adapter.threatinteladapter.end.ts":"1528192727458","server_name":"login.live.com","bro_timestamp":"1216706999.444925","ip_dst_port":443,"enrichmentsplitterbolt.splitter.end.ts":"1528192727442","enrichments.geo.ip_dst_addr.city":"Redmond","subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","enrichments.geo.ip_dst_addr.latitude":"47.6801","cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"enrichmentsplitterbolt.splitter.begin.ts":"1528192727442","adapter.hostfromjsonlistadapter.end.ts":"1528192727444","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5808079","adapter.geoadapter.begin.ts":"1528192727444","issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:
 \/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","enrichments.geo.ip_dst_addr.postalCode":"98052","uid":"CVrS2IBW8gukBClA8","protocol":"ssl","original_string":"SSL | cipher:TLS_RSA_WITH_RC4_128_MD5 established:true server_name:login.live.com id.orig_p:36532 client_cert_chain_fuids:[\"FkYBO41LPAXxh44KFk\",\"FPrzYN1SuBqHflXZId\",\"FZ71xF13r5XVSam1z1\"] subject:CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 id.resp_p:443 cert_chain_fuids:[\"FkYBO41LPAXxh44KFk\",\"FPrzYN1SuBqHflXZId\",\"FZ71xF13r5XVSam1z1\"] version:TLSv10 issuer:CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US uid:CVrS2IBW8gukBClA8 id.orig_h:192.168.15.4 val
 idation_status:unable to get local issuer certificate resumed:false ts:1216706999.444925 id.resp_h:65.54.186.47","ip_dst_addr":"65.54.186.47","threatinteljoinbolt.joiner.ts":"1528192727460","enrichments.geo.ip_dst_addr.dmaCode":"819","enrichmentjoinbolt.joiner.ts":"1528192727447","adapter.hostfromjsonlistadapter.begin.ts":"1528192727444","threatintelsplitterbolt.splitter.begin.ts":"1528192727455","enrichments.geo.ip_dst_addr.longitude":"-122.1206","ip_src_addr":"192.168.15.4","timestamp":1216706999444,"cipher":"TLS_RSA_WITH_RC4_128_MD5","established":true,"client_cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"version":"TLSv10","source.type":"bro","adapter.geoadapter.end.ts":"1528192727444","threatintelsplitterbolt.splitter.end.ts":"1528192727455","adapter.threatinteladapter.begin.ts":"1528192727458","ip_src_port":36532,"enrichments.geo.ip_dst_addr.location_point":"47.6801,-122.1206","guid":"0c5b0898-dbcc-4ac3-a56c-44ade0774e22","validation_status
 ":"unable to get local issuer certificate","resumed":false}
    {"msg":"SSL certificate validation failed with (unable to get local issuer certificate)","suppress_for":3600.0,"adapter.threatinteladapter.end.ts":"1528192727459","note":"SSL::Invalid_Server_Cert","sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US","bro_timestamp":"1216706377.196728","dst":"74.125.19.104","ip_dst_port":443,"enrichmentsplitterbolt.splitter.end.ts":"1528192727443","enrichments.geo.ip_dst_addr.city":"Morganton","enrichments.geo.ip_dst_addr.latitude":"35.7454","dropped":false,"enrichmentsplitterbolt.splitter.begin.ts":"1528192727443","adapter.hostfromjsonlistadapter.end.ts":"1528192727445","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"4480219","adapter.geoadapter.begin.ts":"1528192727445","enrichments.geo.ip_dst_addr.postalCode":"28680","uid":"CNHQmp1mNiZHdAf5Ce","protocol":"notice","original_string":"NOTICE | msg:SSL certificate validation failed with (unable to get local issuer certificate) suppress_for:3600.0
  note:SSL::Invalid_Server_Cert sub:CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US id.orig_p:35736 dst:74.125.19.104 src:192.168.15.4 id.resp_p:443 dropped:false peer_descr:bro p:443 uid:CNHQmp1mNiZHdAf5Ce proto:tcp id.orig_h:192.168.15.4 actions:[\"Notice::ACTION_LOG\",\"Notice::ACTION_ALARM\"] ts:1216706377.196728 id.resp_h:74.125.19.104","ip_dst_addr":"74.125.19.104","threatinteljoinbolt.joiner.ts":"1528192727461","enrichments.geo.ip_dst_addr.dmaCode":"517","enrichmentjoinbolt.joiner.ts":"1528192727454","adapter.hostfromjsonlistadapter.begin.ts":"1528192727445","threatintelsplitterbolt.splitter.begin.ts":"1528192727456","enrichments.geo.ip_dst_addr.longitude":"-81.6848","ip_src_addr":"192.168.15.4","timestamp":1216706377196,"src":"192.168.15.4","peer_descr":"bro","source.type":"bro","p":443,"adapter.geoadapter.end.ts":"1528192727445","threatintelsplitterbolt.splitter.end.ts":"1528192727456","adapter.threatinteladapter.begin.ts":"1528192727459","ip_src_port":3573
 6,"proto":"tcp","enrichments.geo.ip_dst_addr.location_point":"35.7454,-81.6848","guid":"79162baa-4798-4a5f-aae5-5c225a6a2bad","actions":["Notice::ACTION_LOG","Notice::ACTION_ALARM"]}
    {"adapter.threatinteladapter.end.ts":"1528192727460","bro_timestamp":"1216698600.338338","ip_dst_port":10000,"enrichmentsplitterbolt.splitter.end.ts":"1528192727444","enrichments.geo.ip_dst_addr.city":"Holmdel","enrichments.geo.ip_dst_addr.latitude":"40.3754","enrichmentsplitterbolt.splitter.begin.ts":"1528192727444","adapter.hostfromjsonlistadapter.end.ts":"1528192727446","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5099193","adapter.geoadapter.begin.ts":"1528192727446","response_path":["SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000"],"enrichments.geo.ip_dst_addr.postalCode":"07733","uid":"Cl2G2m3bdeE8F9I9ei","trans_depth":0,"protocol":"sip","original_string":"SIP | id.orig_p:1033 method:REGISTER request_body_len:0 id.resp_p:10000 response_path:[\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.
 1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\"] uri:sip:t.voncp.com:10000 call_id:7757a70e218b95730dd2daeaac7d20b1@192.168.1.64 uid:Cl2G2m3bdeE8F9I9ei trans_depth:0 request_from:\"16178766111\" <sip:16178766111@t.voncp.com:10000> request_path:[\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\",\"SIP\\\/2.0\\\/UDP 192.168.1.64:10000\"] id.orig_h:192.168.1.64 request_to:\"16178766111\" <sip:16178766111@t.voncp.com:10000> seq:1761527957 REGISTER user_agent:VDV21 001DD92E4F61 2.8.1_1.4.7 LwooEk3GCD\/bcm001DD92E4F61.xml ts:1216698600.338338 id.resp_h:69.59.232.120","ip_dst_addr":"69.59.232.120","threatinteljoinbolt.joiner.ts":"1528192727463","enrichments.geo.ip_dst_addr.dmaCode":"501","enrichmentjoinbolt.joiner.ts":"1528192727455","adapter.hostfromjsonlistadapter.begin.ts":"1528192727446","threatintelsplitterbolt.splitter.begin.ts":"1528192727458","enrichments.geo.ip_dst_addr.longitude":"-74.1712","request_to"
 :"\"16178766111\" <sip:16178766111@t.voncp.com:10000>","ip_src_addr":"192.168.1.64","seq":"1761527957 REGISTER","user_agent":"VDV21 001DD92E4F61 2.8.1_1.4.7 LwooEk3GCD\/bcm001DD92E4F61.xml","timestamp":1216698600338,"method":"REGISTER","request_body_len":0,"uri":"sip:t.voncp.com:10000","call_id":"7757a70e218b95730dd2daeaac7d20b1@192.168.1.64","source.type":"bro","adapter.geoadapter.end.ts":"1528192727446","request_from":"\"16178766111\" <sip:16178766111@t.voncp.com:10000>","threatintelsplitterbolt.splitter.end.ts":"1528192727458","adapter.threatinteladapter.begin.ts":"1528192727460","ip_src_port":1033,"enrichments.geo.ip_dst_addr.location_point":"40.3754,-74.1712","request_path":["SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000","SIP\/2.0\/UDP 192.168.1.64:10000"],"guid":"403f7e81-12d9-4a0c-a846-fa11b81108fe"}
    ```
    3. Pipe the data to bro
      `cat test-bro.txt | ${HDP_HOME}/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic bro`
    4. Check the Storm worker logs for indexing and verify no exceptions relating to multiValued fields are present.



---