You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:28:39 UTC
[30/42] incubator-spot git commit: added some docstrings
added some docstrings
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/a99404b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/a99404b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/a99404b0
Branch: refs/heads/SPOT-181_ODM
Commit: a99404b05045087bfd02d99f4764df1738959566
Parents: 9215d81
Author: tpltnt <tp...@dropcut.net>
Authored: Thu Jan 25 11:28:26 2018 +0100
Committer: tpltnt <tp...@dropcut.net>
Committed: Thu Jan 25 11:28:26 2018 +0100
----------------------------------------------------------------------
spot-ingest/pipelines/proxy/bluecoat.py | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a99404b0/spot-ingest/pipelines/proxy/bluecoat.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index 31d89ca..1fe02a2 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -62,7 +62,10 @@ proxy_schema = StructType([
StructField("h", StringType(), True)])
def main():
-
+ """
+ Handle commandline arguments and
+ start the collector.
+ """
# input Parameters
parser = argparse.ArgumentParser(description="Bluecoat Parser")
parser.add_argument('-zk','--zookeeper',dest='zk',required=True,help='Zookeeper IP and port (i.e. 10.0.0.1:2181)',metavar='')
@@ -83,7 +86,12 @@ def spot_decoder(s):
return s
def split_log_entry(line):
+ """
+ Split the given line into its fields.
+ :param line: line to split
+ :returns: list
+ """
lex = shlex.shlex(line)
lex.quotes = '"'
lex.whitespace_split = True
@@ -91,7 +99,12 @@ def split_log_entry(line):
return list(lex)
def proxy_parser(proxy_fields):
-
+ """
+ Parse and normalize data.
+
+ :param proxy_fields: list with fields from log
+ :returns: list of str
+ """
proxy_parsed_data = []
if len(proxy_fields) > 1:
@@ -114,7 +127,9 @@ def proxy_parser(proxy_fields):
def save_data(rdd,sqc,db,db_table,topic):
-
+ """
+ Create and save a data frame with the given data.
+ """
if not rdd.isEmpty():
df = sqc.createDataFrame(rdd,proxy_schema)