You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:28:27 UTC
[18/42] incubator-spot git commit: [OA][PEP 8] reformat utils.py
[OA][PEP 8] reformat utils.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/afeb0994
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/afeb0994
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/afeb0994
Branch: refs/heads/SPOT-181_ODM
Commit: afeb0994fe8b46bb510a6947e367391d12930e9e
Parents: 0e74919
Author: natedogs911 <na...@gmail.com>
Authored: Thu Jan 18 11:08:56 2018 -0800
Committer: natedogs911 <na...@gmail.com>
Committed: Thu Jan 18 11:08:56 2018 -0800
----------------------------------------------------------------------
spot-oa/oa/utils.py | 282 +++++++++++++++++++++++------------------------
1 file changed, 136 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/afeb0994/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 2bed10e..8ac6555 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -22,115 +22,114 @@ import csv
import sys
import ConfigParser
+
class Util(object):
-
- @classmethod
- def get_logger(cls,logger_name,create_file=False):
-
-
- # create logger for prd_ci
- log = logging.getLogger(logger_name)
- log.setLevel(level=logging.INFO)
-
- # create formatter and add it to the handlers
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
- if create_file:
- # create file handler for logger.
- fh = logging.FileHandler('oa.log')
- fh.setLevel(level=logging.DEBUG)
- fh.setFormatter(formatter)
- # reate console handler for logger.
- ch = logging.StreamHandler()
- ch.setLevel(level=logging.DEBUG)
- ch.setFormatter(formatter)
-
- # add handlers to logger.
- if create_file:
- log.addHandler(fh)
-
- log.addHandler(ch)
- return log
-
- @classmethod
- def get_spot_conf(cls):
-
- conf_file = "/etc/spot.conf"
- config = ConfigParser.ConfigParser()
- config.readfp(SecHead(open(conf_file)))
-
- return config
-
- @classmethod
- def create_oa_folders(cls,type,date):
-
- # create date and ingest summary folder structure if they don't' exist.
- root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- data_type_folder = "{0}/data/{1}/{2}"
- if not os.path.isdir(data_type_folder.format(root_path,type,date)): os.makedirs(data_type_folder.format(root_path,type,date))
- if not os.path.isdir(data_type_folder.format(root_path,type,"ingest_summary")): os.makedirs(data_type_folder.format(root_path,type,"ingest_summary"))
-
- # create ipynb folders.
- ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path,type,date)
- if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)
-
- # retun path to folders.
- data_path = data_type_folder.format(root_path,type,date)
- ingest_path = data_type_folder.format(root_path,type,"ingest_summary")
- return data_path,ingest_path,ipynb_folder
-
- @classmethod
- def get_ml_results_form_hdfs(cls,hdfs_file_path,local_path):
-
- # get results from hdfs.
- get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path,local_path)
- subprocess.call(get_results_cmd,shell=True)
- return get_results_cmd
-
- @classmethod
- def read_results(cls,file,limit, delimiter=','):
-
- # read csv results.
- result_rows = []
- with open(file, 'rb') as results_file:
- csv_reader = csv.reader(results_file, delimiter = delimiter)
- for i in range(0, int(limit)):
- try:
- row = csv_reader.next()
- except StopIteration:
- return result_rows
- result_rows.append(row)
- return result_rows
-
- @classmethod
- def ip_to_int(self,ip):
-
- try:
- o = map(int, ip.split('.'))
- res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
- return res
-
- except ValueError:
- return None
-
-
- @classmethod
- def create_csv_file(cls,full_path_file,content,delimiter=','):
- with open(full_path_file, 'w+') as u_file:
- writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
- writer.writerows(content)
-
-
- @classmethod
- def cast_val(self,value):
- try:
- val = int(value)
+ @classmethod
+ def get_logger(cls, logger_name, create_file=False):
+
+ # create logger for prd_ci
+ log = logging.getLogger(logger_name)
+ log.setLevel(level=logging.INFO)
+
+ # create formatter and add it to the handlers
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ if create_file:
+ # create file handler for logger.
+ fh = logging.FileHandler('oa.log')
+ fh.setLevel(level=logging.DEBUG)
+ fh.setFormatter(formatter)
+ # reate console handler for logger.
+ ch = logging.StreamHandler()
+ ch.setLevel(level=logging.DEBUG)
+ ch.setFormatter(formatter)
+
+ # add handlers to logger.
+ if create_file:
+ log.addHandler(fh)
+
+ log.addHandler(ch)
+ return log
+
+ @classmethod
+ def get_spot_conf(cls):
+
+ conf_file = "/etc/spot.conf"
+ config = ConfigParser.ConfigParser()
+ config.readfp(SecHead(open(conf_file)))
+
+ return config
+
+ @classmethod
+ def create_oa_folders(cls, type, date):
+
+ # create date and ingest summary folder structure if they don't' exist.
+ root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ data_type_folder = "{0}/data/{1}/{2}"
+ if not os.path.isdir(data_type_folder.format(root_path, type, date)): os.makedirs(
+ data_type_folder.format(root_path, type, date))
+ if not os.path.isdir(data_type_folder.format(root_path, type, "ingest_summary")): os.makedirs(
+ data_type_folder.format(root_path, type, "ingest_summary"))
+
+ # create ipynb folders.
+ ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path, type, date)
+ if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)
+
+ # retun path to folders.
+ data_path = data_type_folder.format(root_path, type, date)
+ ingest_path = data_type_folder.format(root_path, type, "ingest_summary")
+ return data_path, ingest_path, ipynb_folder
+
+ @classmethod
+ def get_ml_results_form_hdfs(cls, hdfs_file_path, local_path):
+
+ # get results from hdfs.
+ get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path, local_path)
+ subprocess.call(get_results_cmd, shell=True)
+ return get_results_cmd
+
+ @classmethod
+ def read_results(cls, file, limit, delimiter=','):
+
+ # read csv results.
+ result_rows = []
+ with open(file, 'rb') as results_file:
+ csv_reader = csv.reader(results_file, delimiter=delimiter)
+ for i in range(0, int(limit)):
+ try:
+ row = csv_reader.next()
+ except StopIteration:
+ return result_rows
+ result_rows.append(row)
+ return result_rows
+
+ @classmethod
+ def ip_to_int(self, ip):
+
+ try:
+ o = map(int, ip.split('.'))
+ res = (16777216 * o[0]) + (65536 * o[1]) + (256 * o[2]) + o[3]
+ return res
+
+ except ValueError:
+ return None
+
+ @classmethod
+ def create_csv_file(cls, full_path_file, content, delimiter=','):
+ with open(full_path_file, 'w+') as u_file:
+ writer = csv.writer(u_file, quoting=csv.QUOTE_NONE, delimiter=delimiter)
+ writer.writerows(content)
+
+ @classmethod
+ def cast_val(self, value):
+ try:
+ val = int(value)
+ except:
+ try:
+ val = float(value)
except:
- try:
- val = float(value)
- except:
- val = str(value)
- return val
+ val = str(value)
+ return val
class SecHead(object):
@@ -140,47 +139,38 @@ class SecHead(object):
def readline(self):
if self.sechead:
- try:
+ try:
return self.sechead
- finally:
+ finally:
self.sechead = None
- else:
+ else:
return self.fp.readline()
-class ProgressBar(object):
-
- def __init__(self,total,prefix='',sufix='',decimals=2,barlength=60):
-
- self._total = total
- self._prefix = prefix
- self._sufix = sufix
- self._decimals = decimals
- self._bar_length = barlength
- self._auto_iteration_status = 0
-
- def start(self):
-
- self._move_progress_bar(0)
-
- def update(self,iterator):
-
- self._move_progress_bar(iterator)
-
- def auto_update(self):
-
- self._auto_iteration_status += 1
- self._move_progress_bar(self._auto_iteration_status)
-
- def _move_progress_bar(self,iteration):
-
- filledLength = int(round(self._bar_length * iteration / float(self._total)))
- percents = round(100.00 * (iteration / float(self._total)), self._decimals)
- bar = '#' * filledLength + '-' * (self._bar_length - filledLength)
- sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix))
- sys.stdout.flush()
-
- if iteration == self._total:print("\n")
-
-
-
+class ProgressBar(object):
+ def __init__(self, total, prefix='', sufix='', decimals=2, barlength=60):
+ self._total = total
+ self._prefix = prefix
+ self._sufix = sufix
+ self._decimals = decimals
+ self._bar_length = barlength
+ self._auto_iteration_status = 0
+
+ def start(self):
+ self._move_progress_bar(0)
+
+ def update(self, iterator):
+ self._move_progress_bar(iterator)
+
+ def auto_update(self):
+ self._auto_iteration_status += 1
+ self._move_progress_bar(self._auto_iteration_status)
+
+ def _move_progress_bar(self, iteration):
+ filledLength = int(round(self._bar_length * iteration / float(self._total)))
+ percents = round(100.00 * (iteration / float(self._total)), self._decimals)
+ bar = '#' * filledLength + '-' * (self._bar_length - filledLength)
+ sys.stdout.write("{0} [{1}] {2}% {3}\r".format(self._prefix, bar, percents, self._sufix))
+ sys.stdout.flush()
+
+ if iteration == self._total: print("\n")