You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/01/25 18:36:52 UTC
[40/49] incubator-spot git commit: OA Quick fix to DNS and Proxy to
skip reputation check, tld check now wont break with unknown dns names
OA Quick fix to DNS and Proxy to skip reputation check, tld check now wont break with unknown dns names
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/c9e27ba0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/c9e27ba0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/c9e27ba0
Branch: refs/heads/master
Commit: c9e27ba0fb59c8ab53e9ecaf7b8001fba88ee3d7
Parents: 7a7d91d
Author: LedaLima <ga...@intel.com>
Authored: Fri Jan 13 18:21:37 2017 -0600
Committer: Everardo Lopez Sandoval (Intel) <el...@elopezsa-mac02.ra.intel.com>
Committed: Fri Jan 20 17:01:02 2017 -0800
----------------------------------------------------------------------
spot-oa/oa/dns/dns_oa.py | 39 +++++++++++++++++++++++++++++----------
spot-oa/oa/proxy/proxy_oa.py | 17 +++++++++++------
spot-oa/oa/utils.py | 8 +++++---
3 files changed, 45 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c9e27ba0/spot-oa/oa/dns/dns_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_oa.py b/spot-oa/oa/dns/dns_oa.py
index 2f8687f..ffcc839 100644
--- a/spot-oa/oa/dns/dns_oa.py
+++ b/spot-oa/oa/dns/dns_oa.py
@@ -143,25 +143,39 @@ class OA(object):
dns_scores_csv = "{0}/dns_scores.csv".format(self._data_path)
dns_scores_final = self._move_time_stamp(self._dns_scores)
dns_scores_final.insert(0,self._dns_scores_headers)
- Util.create_csv_file(dns_scores_csv,dns_scores_final)
+ Util.create_csv_file(dns_scores_csv,dns_scores_final,',',0)
# create bk file
dns_scores_bu_csv = "{0}/dns_scores_bu.csv".format(self._data_path)
- Util.create_csv_file(dns_scores_bu_csv,dns_scores_final)
+ Util.create_csv_file(dns_scores_bu_csv,dns_scores_final,',',0)
def _add_tld_column(self):
qry_name_col = self._conf['dns_results_fields']['dns_qry_name']
- self._dns_scores = [conn + [ get_tld("http://" + str(conn[qry_name_col]), fail_silently=True) if "http://" not in str(conn[qry_name_col]) else get_tld(str(conn[qry_name_col]), fail_silently=True)] for conn in self._dns_scores ]
+ self._dns_scores = [conn + [ self._get_valid_tld(str(conn[qry_name_col])) ] for conn in self._dns_scores ]
+ def _get_valid_tld(self, qry_name):
+ tld = ""
+ try:
+ if "http://" not in qry_name:
+ tld = get_tld("http://" + qry_name)
+ else:
+ tld = get_tld(qry_name)
+ except ValueError:
+ self._logger.error("Unable to get top level domain from query: {0}".format(qry_name))
+ tld = "UNKNOWN"
+ return tld
+
+
def _add_reputation(self):
# read configuration.
reputation_conf_file = "{0}/components/reputation/reputation_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self._logger.info("Reading reputation configuration file: {0}".format(reputation_conf_file))
rep_conf = json.loads(open(reputation_conf_file).read())
-
+
+
# initialize reputation services.
self._rep_services = []
self._logger.info("Initializing reputation services.")
@@ -183,13 +197,18 @@ class OA(object):
# get reputation per column.
self._logger.info("Getting reputation for each service in config")
rep_services_results = []
- for key,value in rep_cols.items():
- rep_services_results = [ rep_service.check(None,value) for rep_service in self._rep_services]
- rep_results = {}
- for result in rep_services_results:
- rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+
+ if self._rep_services :
+ for key,value in rep_cols.items():
+ rep_services_results = [ rep_service.check(None,value) for rep_service in self._rep_services]
+ rep_results = {}
+ for result in rep_services_results:
+ rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
+
+ self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+ else:
+ self._dns_scores = [ conn + [""] for conn in self._dns_scores ]
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c9e27ba0/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 7359ba1..cb13ed3 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -171,14 +171,19 @@ class OA(object):
# get reputation per column.
self._logger.info("Getting reputation for each service in config")
rep_services_results = []
- for key,value in rep_cols.items():
- rep_services_results = [ rep_service.check(None,value,True) for rep_service in self._rep_services]
- rep_results = {}
+ if self._rep_services :
+ for key,value in rep_cols.items():
+ rep_services_results = [ rep_service.check(None,value,True) for rep_service in self._rep_services]
+ rep_results = {}
+
+ for result in rep_services_results:
+ rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
+
+ self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ else:
+ self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
- for result in rep_services_results:
- rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
def _add_severity(self):
# Add severity column
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c9e27ba0/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 553315a..52b251a 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -96,13 +96,15 @@ class Util(object):
except ValueError:
return None
+
@classmethod
- def create_csv_file(cls,full_path_file,content,delimiter=','):
-
+ def create_csv_file(cls,full_path_file,content,delimiter=',',set_quoting=3):
+ #set_quoting: 0 - MINIMAL, 1 - ALL, 3 - NONE
with open(full_path_file, 'w+') as u_file:
- writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
+ writer = csv.writer(u_file, quoting=set_quoting, quotechar='"', delimiter=delimiter)
writer.writerows(content)
+
class SecHead(object):
def __init__(self, fp):
self.fp = fp