You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2016/12/08 18:49:06 UTC

[26/50] [abbrv] incubator-spot git commit: Adding ingest summary functionality to OA module

Adding ingest summary functionality to OA module


Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/3cb7dbe3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/3cb7dbe3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/3cb7dbe3

Branch: refs/heads/master
Commit: 3cb7dbe38e11a5f4dd43aaaaf3e90490c18dda30
Parents: 9763e2a
Author: elopezsa <ev...@intel.com>
Authored: Wed Nov 30 09:46:58 2016 -0600
Committer: elopezsa <ev...@intel.com>
Committed: Wed Nov 30 09:46:58 2016 -0600

----------------------------------------------------------------------
 spot-ingest/README.md      | 21 +++++++++++----
 spot-oa/oa/flow/flow_oa.py | 58 ++++++++++++++++++++++++++++++++++-------
 2 files changed, 64 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/3cb7dbe3/spot-ingest/README.md
----------------------------------------------------------------------
diff --git a/spot-ingest/README.md b/spot-ingest/README.md
index 193ea9f..6239844 100644
--- a/spot-ingest/README.md
+++ b/spot-ingest/README.md
@@ -22,7 +22,7 @@ Ingest data is captured or transferred into the Hadoop cluster, where they are t
 
 Ingest framework needs Kafka to work in real-time streaming. Add Kafka service using Cloudera Manager. If you are using a Cloudera Manager version < 5.4.1 you will need to add the kafka parcel manually.
 
-Ingest module uses a default configuration for the message size (999999 bytes), if you modify this size in the ingest configuration file you will need to modify the following configuration properties in kafka:
+Ingest module uses a default configuration for the message size (900000 bytes), if you modify this size in the ingest configuration file you will need to modify the following configuration properties in kafka:
 
 * message.max.bytes
 * replica.fetch.max.bytes
@@ -48,19 +48,30 @@ The file **ingest_conf.json** contains all the required configuration to start t
 *  **dbname:** Name of HIVE database where all the ingested data will be stored in avro-parquet format.
 *  **hdfs_app_path:** Application path in HDFS where the pipelines will be stored (i.e /user/_application_user_/). 
 *  **kafka:** Kafka and Zookeeper server information required to create/listen topics and partitions.
+*  **collector_processes:** Ingest framework uses  multiprocessing to collect files (different from workers), this configuration key defines the numbers of collector processes to use.
+*  **spark-streaming:** Proxy pipeline uses spark streaming to ingest data, this configuration is required to setup the spark application for more details please check : [how to configure spark](https://github.com/Open-Network-Insight/open-network-insight/blob/spot/spot-ml/SPARKCONF.md)
 *  **pipelines:** In this section you can add multiple configurations for either the same pipeline or different pipelines. The configuration name must be lowercase without spaces (i.e. flow_internals).
 
 **Configuration example:**
 
-      "dbname" : "database name",
-      "hdfs_app_path" : "hdfs application path",
+     "dbname" : "database name",
+     "hdfs_app_path" : "hdfs application path",
+     "collector_processes":5,
+     "ingestion_interval":1,
+     "spark-streaming":{
+                "driver_memory":"",
+                "spark_exec":"",
+                "spark_executor_memory":"",
+                "spark_executor_cores":"",
+                "spark_batch_size":""
+      },
       "kafka":{
             "kafka_server":"kafka ip",
             "kafka_port":"kafka port",
             "zookeper_server":"zk ip",
             "zookeper_port":"zk port",
-            "message_size":999999
-       },
+            "message_size":900000
+        },
       "pipelines":{
       
          "flow_internals":{

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/3cb7dbe3/spot-oa/oa/flow/flow_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/flow_oa.py b/spot-oa/oa/flow/flow_oa.py
index 1fbd503..50ecf9a 100644
--- a/spot-oa/oa/flow/flow_oa.py
+++ b/spot-oa/oa/flow/flow_oa.py
@@ -284,17 +284,17 @@ class OA(object):
 
     def _get_oa_details(self):
 
-	self._logger.info("Getting OA Flow suspicious details/chord diagram")
-	# start suspicious connects details process.
-	p_sp = Process(target=self._get_suspicious_details)
-	p_sp.start()
+        self._logger.info("Getting OA Flow suspicious details/chord diagram")
+        # start suspicious connects details process.
+        p_sp = Process(target=self._get_suspicious_details)
+        p_sp.start()
 
-	# start chord diagram process.
-	p_ch = Process(target=self._get_chord_details)
-	p_ch.start()
+        # start chord diagram process.
+        p_ch = Process(target=self._get_chord_details)
+        p_ch.start()
 
-	p_sp.join()
-	p_ch.join()
+        p_sp.join()
+        p_ch.join()
 
 	
     def _get_suspicious_details(self,bar=None):
@@ -385,6 +385,44 @@ class OA(object):
                     self._engine.query(ch_query.format(self._db,self._table_name,yr,mn,dy,ip,ips_filter),chord_file,delimiter="\\t")
 
      
- 
+    def _ingest_summary(self):
+        
+        # get date parameters.
+        yr = self._date[:4]
+        mn = self._date[4:6]
+        dy = self._date[6:]
 
+        # get ingest summary.           
+        ingest_summary_qry = ("SELECT tryear, trmonth, trday, trhour, trminute, COUNT(*) flows"
+                              " FROM onidb.flow "
+                              " WHERE "
+                              " y={0} "
+                              " AND m={1} "
+                              " AND d={2} "
+                              " AND unix_tstamp IS NOT NULL "
+                              " GROUP BY tryear, trmonth, trday, trhour, trminute;")
+
+        ingest_summary_qry = ingest_summary_qry.format(yr,mn,dy)
+
+        results_file = "{0}/results_{1}.csv".format(self._data_path,self._date)
+        self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
+        
+        result_rows = []        
+        with open(results_file, 'rb') as rf:
+            csv_reader = csv.reader(rf, delimiter = ",")
+            result_rows = list(csv_reader)
+        
+        result_rows = iter(result_rows)
+        next(result_rows)
+        ingest_summary_results2 = [ ["{0}-{1}-{2} {3}:{4}".format(yr,mn,dy,row[3],row[4]), row[5]] for row in result_rows ]
+        ingest_summay_file = "{0}/is_{1}.csv".format(self._data_path,self._date)
+
+        write_format =  'a' if os.path.isfile(ingest_summay_file) else 'w+'
+        with open(ingest_summay_file, write_format) as u_file:
+            writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=",")
+            writer.writerows(ingest_summary_results2)
+
+        rm_big_file = "rm {0}".format(results_file)
+        os.remove(results_file)
+