You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/01/21 00:41:35 UTC
[1/2] incubator-spot git commit: Reverted changes to split into two
different PRs
Repository: incubator-spot
Updated Branches:
refs/heads/master ce10f2053 -> 78387cf13
Reverted changes to split into two different PRs
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/78387cf1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/78387cf1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/78387cf1
Branch: refs/heads/master
Commit: 78387cf13b20f4acd1f00f44032cbe06d9daa2dd
Parents: 10fecf7
Author: LedaLima <ga...@intel.com>
Authored: Fri Jan 20 14:28:18 2017 -0600
Committer: everlosa <ev...@intel.com>
Committed: Fri Jan 20 16:40:41 2017 -0800
----------------------------------------------------------------------
spot-oa/oa/dns/dns_oa.py | 22 ++++------------------
spot-oa/oa/utils.py | 5 ++---
2 files changed, 6 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/78387cf1/spot-oa/oa/dns/dns_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_oa.py b/spot-oa/oa/dns/dns_oa.py
index ffcc839..abb8cac 100644
--- a/spot-oa/oa/dns/dns_oa.py
+++ b/spot-oa/oa/dns/dns_oa.py
@@ -143,31 +143,17 @@ class OA(object):
dns_scores_csv = "{0}/dns_scores.csv".format(self._data_path)
dns_scores_final = self._move_time_stamp(self._dns_scores)
dns_scores_final.insert(0,self._dns_scores_headers)
- Util.create_csv_file(dns_scores_csv,dns_scores_final,',',0)
+ Util.create_csv_file(dns_scores_csv,dns_scores_final)
# create bk file
dns_scores_bu_csv = "{0}/dns_scores_bu.csv".format(self._data_path)
- Util.create_csv_file(dns_scores_bu_csv,dns_scores_final,',',0)
+ Util.create_csv_file(dns_scores_bu_csv,dns_scores_final)
def _add_tld_column(self):
- qry_name_col = self._conf['dns_results_fields']['dns_qry_name']
- self._dns_scores = [conn + [ self._get_valid_tld(str(conn[qry_name_col])) ] for conn in self._dns_scores ]
-
+ qry_name_col = self._conf['dns_results_fields']['dns_qry_name']
+ self._dns_scores = [conn + [ get_tld("http://" + str(conn[qry_name_col]), fail_silently=True) if "http://" not in str(conn[qry_name_col]) else get_tld(str(conn[qry_name_col]), fail_silently=True)] for conn in self._dns_scores ]
- def _get_valid_tld(self, qry_name):
- tld = ""
- try:
- if "http://" not in qry_name:
- tld = get_tld("http://" + qry_name)
- else:
- tld = get_tld(qry_name)
- except ValueError:
- self._logger.error("Unable to get top level domain from query: {0}".format(qry_name))
- tld = "UNKNOWN"
- return tld
-
-
def _add_reputation(self):
# read configuration.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/78387cf1/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 52b251a..d08b26e 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -98,10 +98,9 @@ class Util(object):
@classmethod
- def create_csv_file(cls,full_path_file,content,delimiter=',',set_quoting=3):
- #set_quoting: 0 - MINIMAL, 1 - ALL, 3 - NONE
+ def create_csv_file(cls,full_path_file,content,delimiter=','):
with open(full_path_file, 'w+') as u_file:
- writer = csv.writer(u_file, quoting=set_quoting, quotechar='"', delimiter=delimiter)
+ writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
writer.writerows(content)
[2/2] incubator-spot git commit: OA Quick fix to DNS and Proxy to
skip reputation check, tld check now wont break with unknown dns names
Posted by ev...@apache.org.
OA Quick fix to DNS and Proxy to skip reputation check, tld check now wont break with unknown dns names
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/10fecf70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/10fecf70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/10fecf70
Branch: refs/heads/master
Commit: 10fecf70bd5dda3c82a7c94b3360e52d5f283cf5
Parents: ce10f20
Author: LedaLima <ga...@intel.com>
Authored: Fri Jan 13 18:21:37 2017 -0600
Committer: everlosa <ev...@intel.com>
Committed: Fri Jan 20 16:40:41 2017 -0800
----------------------------------------------------------------------
spot-oa/oa/dns/dns_oa.py | 39 +++++++++++++++++++++++++++++----------
spot-oa/oa/proxy/proxy_oa.py | 17 +++++++++++------
spot-oa/oa/utils.py | 8 +++++---
3 files changed, 45 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/10fecf70/spot-oa/oa/dns/dns_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_oa.py b/spot-oa/oa/dns/dns_oa.py
index 2f8687f..ffcc839 100644
--- a/spot-oa/oa/dns/dns_oa.py
+++ b/spot-oa/oa/dns/dns_oa.py
@@ -143,25 +143,39 @@ class OA(object):
dns_scores_csv = "{0}/dns_scores.csv".format(self._data_path)
dns_scores_final = self._move_time_stamp(self._dns_scores)
dns_scores_final.insert(0,self._dns_scores_headers)
- Util.create_csv_file(dns_scores_csv,dns_scores_final)
+ Util.create_csv_file(dns_scores_csv,dns_scores_final,',',0)
# create bk file
dns_scores_bu_csv = "{0}/dns_scores_bu.csv".format(self._data_path)
- Util.create_csv_file(dns_scores_bu_csv,dns_scores_final)
+ Util.create_csv_file(dns_scores_bu_csv,dns_scores_final,',',0)
def _add_tld_column(self):
qry_name_col = self._conf['dns_results_fields']['dns_qry_name']
- self._dns_scores = [conn + [ get_tld("http://" + str(conn[qry_name_col]), fail_silently=True) if "http://" not in str(conn[qry_name_col]) else get_tld(str(conn[qry_name_col]), fail_silently=True)] for conn in self._dns_scores ]
+ self._dns_scores = [conn + [ self._get_valid_tld(str(conn[qry_name_col])) ] for conn in self._dns_scores ]
+ def _get_valid_tld(self, qry_name):
+ tld = ""
+ try:
+ if "http://" not in qry_name:
+ tld = get_tld("http://" + qry_name)
+ else:
+ tld = get_tld(qry_name)
+ except ValueError:
+ self._logger.error("Unable to get top level domain from query: {0}".format(qry_name))
+ tld = "UNKNOWN"
+ return tld
+
+
def _add_reputation(self):
# read configuration.
reputation_conf_file = "{0}/components/reputation/reputation_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self._logger.info("Reading reputation configuration file: {0}".format(reputation_conf_file))
rep_conf = json.loads(open(reputation_conf_file).read())
-
+
+
# initialize reputation services.
self._rep_services = []
self._logger.info("Initializing reputation services.")
@@ -183,13 +197,18 @@ class OA(object):
# get reputation per column.
self._logger.info("Getting reputation for each service in config")
rep_services_results = []
- for key,value in rep_cols.items():
- rep_services_results = [ rep_service.check(None,value) for rep_service in self._rep_services]
- rep_results = {}
- for result in rep_services_results:
- rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+
+ if self._rep_services :
+ for key,value in rep_cols.items():
+ rep_services_results = [ rep_service.check(None,value) for rep_service in self._rep_services]
+ rep_results = {}
+ for result in rep_services_results:
+ rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
+
+ self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+ else:
+ self._dns_scores = [ conn + [""] for conn in self._dns_scores ]
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/10fecf70/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 7359ba1..cb13ed3 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -171,14 +171,19 @@ class OA(object):
# get reputation per column.
self._logger.info("Getting reputation for each service in config")
rep_services_results = []
- for key,value in rep_cols.items():
- rep_services_results = [ rep_service.check(None,value,True) for rep_service in self._rep_services]
- rep_results = {}
+ if self._rep_services :
+ for key,value in rep_cols.items():
+ rep_services_results = [ rep_service.check(None,value,True) for rep_service in self._rep_services]
+ rep_results = {}
+
+ for result in rep_services_results:
+ rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
+
+ self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ else:
+ self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
- for result in rep_services_results:
- rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
def _add_severity(self):
# Add severity column
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/10fecf70/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 553315a..52b251a 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -96,13 +96,15 @@ class Util(object):
except ValueError:
return None
+
@classmethod
- def create_csv_file(cls,full_path_file,content,delimiter=','):
-
+ def create_csv_file(cls,full_path_file,content,delimiter=',',set_quoting=3):
+ #set_quoting: 0 - MINIMAL, 1 - ALL, 3 - NONE
with open(full_path_file, 'w+') as u_file:
- writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
+ writer = csv.writer(u_file, quoting=set_quoting, quotechar='"', delimiter=delimiter)
writer.writerows(content)
+
class SecHead(object):
def __init__(self, fp):
self.fp = fp