You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/01/25 18:36:42 UTC
[30/49] incubator-spot git commit: merging in recent changes to spot
branch to remove merge conflicts Merge branch 'spot' of
https://github.com/Open-Network-Insight/open-network-insight into
User_Domain_Designation_Fix
merging in recent changes to spot branch to remove merge conflicts
Merge branch 'spot' of https://github.com/Open-Network-Insight/open-network-insight into User_Domain_Designation_Fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/06900bb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/06900bb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/06900bb6
Branch: refs/heads/master
Commit: 06900bb643ce3656498a103b2b5510b04f3711de
Parents: 1a0269f 760dbf3
Author: Brandon Edwards <br...@intel.com>
Authored: Thu Dec 22 15:12:11 2016 -0800
Committer: Brandon Edwards <br...@intel.com>
Committed: Thu Dec 22 15:12:11 2016 -0800
----------------------------------------------------------------------
spot-ml/README.md | 24 +--
spot-ml/install_ml.sh | 12 --
spot-ml/ml_ops.sh | 30 +--
spot-ml/ml_test.sh | 30 +--
.../org/apache/spot/SuspiciousConnects.scala | 19 +-
.../spot/SuspiciousConnectsScoreFunction.scala | 19 +-
.../scala/org/apache/spot/dns/DNSSchema.scala | 2 -
.../dns/DNSSuspiciousConnectsAnalysis.scala | 172 +++++++++++----
.../org/apache/spot/dns/DNSWordCreation.scala | 28 ++-
.../dns/model/DNSSuspiciousConnectsModel.scala | 99 +++++++--
.../FlowSuspiciousConnectsAnalysis.scala | 119 ++++++++--
.../apache/spot/netflow/FlowWordCreator.scala | 65 +++---
.../spot/netflow/model/FlowScoreFunction.scala | 9 +-
.../model/FlowSuspiciousConnectsModel.scala | 73 ++++---
.../org/apache/spot/proxy/ProxySchema.scala | 49 +++++
.../proxy/ProxySuspiciousConnectsAnalysis.scala | 155 +++++++++++--
.../proxy/ProxySuspiciousConnectsModel.scala | 65 ++++--
.../apache/spot/proxy/ProxyWordCreation.scala | 27 ++-
.../utilities/data/InputOutputDataHandler.scala | 63 ++++++
.../data/validation/InvalidDataHandler.scala | 56 +++++
.../org/apache/spot/DNSWordCreationTest.scala | 21 --
.../org/apache/spot/FlowWordCreatorTest.scala | 216 -------------------
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 114 ++++++++++
.../apache/spot/dns/DNSWordCreationTest.scala | 17 ++
.../FlowSuspiciousCoonectsAnalysis.scala | 125 +++++++++++
.../spot/netflow/FlowWordCreatorTest.scala | 214 ++++++++++++++++++
.../ProxySuspiciousConnectsAnalysisTest.scala | 135 ++++++++++++
spot-oa/oa/components/iana/dns-qclass.csv | 1 +
spot-oa/oa/components/iana/dns-qtype.csv | 1 +
spot-oa/oa/components/iana/dns-rcode.csv | 1 +
spot-setup/spot.conf | 13 +-
31 files changed, 1435 insertions(+), 539 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/06900bb6/spot-ml/ml_ops.sh
----------------------------------------------------------------------
diff --cc spot-ml/ml_ops.sh
index 02a9c1d,a3406e8..1ff129e
--- a/spot-ml/ml_ops.sh
+++ b/spot-ml/ml_ops.sh
@@@ -45,31 -45,9 +45,17 @@@ els
RAWDATA_PATH=${PROXY_PATH}
fi
+# pass the user domain designation if not empty
+
+if [ ! -z $USER_DOMAIN ] ; then
+ USER_DOMAIN_CMD="--userdomain $USER_DOMAIN"
+else
+ USER_DOMAIN_CMD=''
+fi
+
FEEDBACK_PATH=${LPATH}/${DSOURCE}_scores.csv
- DUPFACTOR=1000
-
- PREPROCESS_STEP=${DSOURCE}_pre_lda
- POSTPROCESS_STEP=${DSOURCE}_post_lda
-
- HDFS_WORDCOUNTS=${HPATH}/word_counts
-
- # paths for intermediate files
- HDFS_DOCRESULTS=${HPATH}/doc_results.csv
- LOCAL_DOCRESULTS=${LPATH}/doc_results.csv
-
- HDFS_WORDRESULTS=${HPATH}/word_results.csv
- LOCAL_WORDRESULTS=${LPATH}/word_results.csv
HDFS_SCORED_CONNECTS=${HPATH}/scores
- HDFS_MODEL=${HPATH}/model
LDA_OUTPUT_DIR=${DSOURCE}/${FDATE}
@@@ -108,12 -78,5 +86,6 @@@ time spark-submit --class "org.apache.s
--scored ${HDFS_SCORED_CONNECTS} \
--threshold ${TOL} \
--maxresults ${MAXRESULTS} \
- --ldamaxiterations 20
+ --ldamaxiterations 20 \
+ $USER_DOMAIN_CMD
+
- wait
-
- # move results to hdfs.
- cd ${LPATH}
- hadoop fs -getmerge ${HDFS_SCORED_CONNECTS}/part-* ${DSOURCE}_results.csv && hadoop fs -moveFromLocal \
- ${DSOURCE}_results.csv ${HDFS_SCORED_CONNECTS}/${DSOURCE}_results.csv