You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2016/12/08 18:49:06 UTC
[26/50] [abbrv] incubator-spot git commit: Adding ingest summary
functionality to OA module
Adding ingest summary functionality to OA module
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/3cb7dbe3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/3cb7dbe3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/3cb7dbe3
Branch: refs/heads/master
Commit: 3cb7dbe38e11a5f4dd43aaaaf3e90490c18dda30
Parents: 9763e2a
Author: elopezsa <ev...@intel.com>
Authored: Wed Nov 30 09:46:58 2016 -0600
Committer: elopezsa <ev...@intel.com>
Committed: Wed Nov 30 09:46:58 2016 -0600
----------------------------------------------------------------------
spot-ingest/README.md | 21 +++++++++++----
spot-oa/oa/flow/flow_oa.py | 58 ++++++++++++++++++++++++++++++++++-------
2 files changed, 64 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/3cb7dbe3/spot-ingest/README.md
----------------------------------------------------------------------
diff --git a/spot-ingest/README.md b/spot-ingest/README.md
index 193ea9f..6239844 100644
--- a/spot-ingest/README.md
+++ b/spot-ingest/README.md
@@ -22,7 +22,7 @@ Ingest data is captured or transferred into the Hadoop cluster, where they are t
Ingest framework needs Kafka to work in real-time streaming. Add Kafka service using Cloudera Manager. If you are using a Cloudera Manager version < 5.4.1 you will need to add the kafka parcel manually.
-Ingest module uses a default configuration for the message size (999999 bytes), if you modify this size in the ingest configuration file you will need to modify the following configuration properties in kafka:
+Ingest module uses a default configuration for the message size (900000 bytes), if you modify this size in the ingest configuration file you will need to modify the following configuration properties in kafka:
* message.max.bytes
* replica.fetch.max.bytes
@@ -48,19 +48,30 @@ The file **ingest_conf.json** contains all the required configuration to start t
* **dbname:** Name of HIVE database where all the ingested data will be stored in avro-parquet format.
* **hdfs_app_path:** Application path in HDFS where the pipelines will be stored (i.e /user/_application_user_/).
* **kafka:** Kafka and Zookeeper server information required to create/listen topics and partitions.
+* **collector_processes:** Ingest framework uses multiprocessing to collect files (different from workers), this configuration key defines the numbers of collector processes to use.
+* **spark-streaming:** Proxy pipeline uses spark streaming to ingest data, this configuration is required to setup the spark application for more details please check : [how to configure spark](https://github.com/Open-Network-Insight/open-network-insight/blob/spot/spot-ml/SPARKCONF.md)
* **pipelines:** In this section you can add multiple configurations for either the same pipeline or different pipelines. The configuration name must be lowercase without spaces (i.e. flow_internals).
**Configuration example:**
- "dbname" : "database name",
- "hdfs_app_path" : "hdfs application path",
+ "dbname" : "database name",
+ "hdfs_app_path" : "hdfs application path",
+ "collector_processes":5,
+ "ingestion_interval":1,
+ "spark-streaming":{
+ "driver_memory":"",
+ "spark_exec":"",
+ "spark_executor_memory":"",
+ "spark_executor_cores":"",
+ "spark_batch_size":""
+ },
"kafka":{
"kafka_server":"kafka ip",
"kafka_port":"kafka port",
"zookeper_server":"zk ip",
"zookeper_port":"zk port",
- "message_size":999999
- },
+ "message_size":900000
+ },
"pipelines":{
"flow_internals":{
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/3cb7dbe3/spot-oa/oa/flow/flow_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/flow_oa.py b/spot-oa/oa/flow/flow_oa.py
index 1fbd503..50ecf9a 100644
--- a/spot-oa/oa/flow/flow_oa.py
+++ b/spot-oa/oa/flow/flow_oa.py
@@ -284,17 +284,17 @@ class OA(object):
def _get_oa_details(self):
- self._logger.info("Getting OA Flow suspicious details/chord diagram")
- # start suspicious connects details process.
- p_sp = Process(target=self._get_suspicious_details)
- p_sp.start()
+ self._logger.info("Getting OA Flow suspicious details/chord diagram")
+ # start suspicious connects details process.
+ p_sp = Process(target=self._get_suspicious_details)
+ p_sp.start()
- # start chord diagram process.
- p_ch = Process(target=self._get_chord_details)
- p_ch.start()
+ # start chord diagram process.
+ p_ch = Process(target=self._get_chord_details)
+ p_ch.start()
- p_sp.join()
- p_ch.join()
+ p_sp.join()
+ p_ch.join()
def _get_suspicious_details(self,bar=None):
@@ -385,6 +385,44 @@ class OA(object):
self._engine.query(ch_query.format(self._db,self._table_name,yr,mn,dy,ip,ips_filter),chord_file,delimiter="\\t")
-
+ def _ingest_summary(self):
+
+ # get date parameters.
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ # get ingest summary.
+ ingest_summary_qry = ("SELECT tryear, trmonth, trday, trhour, trminute, COUNT(*) flows"
+ " FROM onidb.flow "
+ " WHERE "
+ " y={0} "
+ " AND m={1} "
+ " AND d={2} "
+ " AND unix_tstamp IS NOT NULL "
+ " GROUP BY tryear, trmonth, trday, trhour, trminute;")
+
+ ingest_summary_qry = ingest_summary_qry.format(yr,mn,dy)
+
+ results_file = "{0}/results_{1}.csv".format(self._data_path,self._date)
+ self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
+
+ result_rows = []
+ with open(results_file, 'rb') as rf:
+ csv_reader = csv.reader(rf, delimiter = ",")
+ result_rows = list(csv_reader)
+
+ result_rows = iter(result_rows)
+ next(result_rows)
+ ingest_summary_results2 = [ ["{0}-{1}-{2} {3}:{4}".format(yr,mn,dy,row[3],row[4]), row[5]] for row in result_rows ]
+ ingest_summay_file = "{0}/is_{1}.csv".format(self._data_path,self._date)
+
+ write_format = 'a' if os.path.isfile(ingest_summay_file) else 'w+'
+ with open(ingest_summay_file, write_format) as u_file:
+ writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=",")
+ writer.writerows(ingest_summary_results2)
+
+ rm_big_file = "rm {0}".format(results_file)
+ os.remove(results_file)
+