You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/02/09 17:43:49 UTC
[4/7] incubator-spot git commit: Merge branch 'SPOT-181_ODM' of
github.com:curtishoward/incubator-spot into SPOT-180_HiveTablesAccess
Merge branch 'SPOT-181_ODM' of github.com:curtishoward/incubator-spot into SPOT-180_HiveTablesAccess
Bringing branch up to date with apache/incubator-spot SPOT-181_ODM
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/6aa2c66d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/6aa2c66d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/6aa2c66d
Branch: refs/heads/SPOT-181_ODM
Commit: 6aa2c66d18990151ec85e1b78450074c9d8342a9
Parents: 1241398 d20c5bf
Author: Curtis Howard <cu...@curtis-MBP.local>
Authored: Thu Jan 25 10:06:16 2018 -0500
Committer: Curtis Howard <cu...@curtis-MBP.local>
Committed: Thu Jan 25 10:06:16 2018 -0500
----------------------------------------------------------------------
LICENSE | 97 +
README.md | 2 +-
dev/release/.rat-excludes | 11 +
dev/release/README.md | 474 +
docs/label-bug-enhancement.bmp | Bin 29390 -> 0 bytes
docs/label-discussion.bmp | Bin 13074 -> 0 bytes
docs/label-issue-type.bmp | Bin 161250 -> 0 bytes
docs/label-repos.bmp | Bin 245838 -> 0 bytes
docs/label-tracking.bmp | Bin 181638 -> 0 bytes
docs/labels-design-tracking.bmp | Bin 181638 -> 0 bytes
docs/oni-approach.png | Bin 58589 -> 0 bytes
docs/oni-guy.png | Bin 34146 -> 0 bytes
docs/oni-guy_big.png | Bin 90430 -> 0 bytes
docs/oni-logo.png | Bin 16768 -> 0 bytes
docs/open-data-model.md | 3310 +
docs/open-data-model/open-data-model.md | 892 -
spot-ingest/common/__init__.py | 16 +
spot-ingest/pipelines/__init__.py | 17 +-
spot-ingest/pipelines/dns/__init__.py | 16 +
.../pipelines/dns/load_dns_avro_parquet.hql | 15 +
spot-ingest/pipelines/flow/__init__.py | 16 +
.../pipelines/flow/load_flow_avro_parquet.hql | 15 +
spot-ingest/pipelines/proxy/__init__.py | 16 +
spot-ingest/pipelines/proxy/bluecoat.py | 2 +-
spot-ingest/streamsets/README.md | 27 +
.../ODMCentrifyIdentityPlatformEventTCP.json | 1096 +
spot-ingest/streamsets/images/ImportContext.png | Bin 0 -> 61789 bytes
.../streamsets/images/ImportPipeline.png | Bin 0 -> 65915 bytes
.../streamsets/netflow/NetFlowODMandLegacy.json | 1463 +
.../qualys/ODMQualysVulnerabilityContext.json | 1276 +
.../qualys/ODMQualysVulnerabilityEvents.json | 1245 +
.../streamsets/windows/ODMWindowsEventLogs.json | 943 +
.../streamsets/windows/WindowsHTTPEdge.json | 603 +
spot-ml/DATA_SAMPLE.md | 57 +
spot-ml/INSTALL.md | 2 +-
spot-ml/ML_OPS.md | 8 +-
spot-ml/README.md | 17 +-
spot-ml/SPOT-ML-JAR.md | 4 +-
spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md | 4 +-
spot-ml/build.sbt | 18 +-
spot-ml/ml_ops.sh | 7 +-
spot-ml/ml_test.sh | 6 +-
spot-ml/project/build.properties | 1 +
spot-ml/project/plugins.sbt | 17 +
.../dns/DNSSuspiciousConnectsAnalysis.scala | 62 +-
.../org/apache/spot/dns/model/DNSFeedback.scala | 14 +-
.../dns/model/DNSSuspiciousConnectsModel.scala | 28 -
.../FlowSuspiciousConnectsAnalysis.scala | 51 +-
.../spot/netflow/model/FlowFeedback.scala | 21 +-
.../model/FlowSuspiciousConnectsModel.scala | 2 +-
.../org/apache/spot/proxy/ProxyFeedback.scala | 28 +-
.../proxy/ProxySuspiciousConnectsAnalysis.scala | 60 +-
.../proxy/ProxySuspiciousConnectsModel.scala | 18 +-
.../org/apache/spot/utilities/TopDomains.scala | 3 +-
.../utilities/data/validation/InputSchema.scala | 51 +
spot-ml/src/test/resources/log4j.properties | 18 +-
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 56 +-
.../dns/DNSSuspiciousConnectsModelTest.scala | 17 +
.../org/apache/spot/netflow/FlowRecord.scala | 35 +
.../FlowSuspiciousConnectsAnalysisTest.scala | 56 +-
.../model/FlowSuspiciousConnectsModelTest.scala | 17 +
.../ProxySuspiciousConnectsAnalysisTest.scala | 87 +-
.../FloatPointPrecisionUtility32Test.scala | 17 +
.../spot/utilities/FloatingPointUtility64.scala | 17 +
.../data/validation/InputSchemaTest.scala | 53 +
spot-ml/top-1m.csv | 1000000 ---------------
spot-oa/README.md | 10 +-
spot-oa/__init__.py | 16 +
spot-oa/api/__init__.py | 16 +
spot-oa/api/graphql/README.md | 13 +
spot-oa/api/graphql/__init__.py | 18 +
spot-oa/api/graphql/common.py | 142 +
spot-oa/api/graphql/dns/__init__.py | 18 +
spot-oa/api/graphql/dns/mutation.py | 182 +
spot-oa/api/graphql/dns/query.py | 519 +
spot-oa/api/graphql/flow/__init__.py | 18 +
spot-oa/api/graphql/flow/mutation.py | 218 +
spot-oa/api/graphql/flow/query.py | 691 +
spot-oa/api/graphql/proxy/__init__.py | 18 +
spot-oa/api/graphql/proxy/mutation.py | 221 +
spot-oa/api/graphql/proxy/query.py | 582 +
spot-oa/api/graphql/schema.py | 69 +
spot-oa/api/graphql/webapp.py | 57 +
spot-oa/api/resources/README.md | 49 +
spot-oa/api/resources/__init__.py | 16 +
spot-oa/api/resources/configurator.py | 59 +
spot-oa/api/resources/dns.py | 395 +
spot-oa/api/resources/flow.py | 958 +
spot-oa/api/resources/hdfs_client.py | 76 +
spot-oa/api/resources/impala_engine.py | 50 +
spot-oa/api/resources/proxy.py | 467 +
spot-oa/arcadia/README.md | 84 +
spot-oa/arcadia/spot_app.json | 1 +
spot-oa/context/README.md | 6 +-
spot-oa/ipython/README | 10 +
spot-oa/ipython/extensions/__init__.py | 16 +
spot-oa/ipython/extensions/spot_webapp.py | 22 +
.../profile_spot/ipython_notebook_config.py | 570 +
spot-oa/ipython/profile_spot/startup/README | 11 +
spot-oa/ipython/profile_spot/startup/graphql.py | 90 +
.../profile_spot/static/custom/ajax-loader.gif | Bin 0 -> 723 bytes
.../profile_spot/static/custom/custom.css | 163 +
.../profile_spot/static/custom/custom.js | 351 +
spot-oa/oa/INSTALL.md | 10 +-
spot-oa/oa/components/README.md | 10 +-
spot-oa/oa/components/__init__.py | 16 +
spot-oa/oa/components/data/__init__.py | 16 +
spot-oa/oa/components/geoloc/__init__.py | 16 +
spot-oa/oa/components/iana/__init__.py | 16 +
spot-oa/oa/components/iana/iana_transform.py | 8 +-
spot-oa/oa/components/nc/__init__.py | 16 +
spot-oa/oa/components/reputation/README.md | 2 +-
spot-oa/oa/components/reputation/__init__.py | 16 +
spot-oa/oa/components/reputation/fb/__init__.py | 16 +
spot-oa/oa/components/reputation/fb/fb.py | 2 +-
.../oa/components/reputation/gti/__init__.py | 16 +
spot-oa/oa/components/reputation/gti/gti.py | 1 +
spot-oa/oa/dns/README.md | 131 +-
spot-oa/oa/dns/__init__.py | 16 +
spot-oa/oa/dns/dns_conf.json | 36 +-
.../ipynb_templates/Advanced_Mode_master.ipynb | 292 +
spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md | 76 -
.../Edge_Investigation_master.ipynb | 269 -
.../dns/ipynb_templates/ThreatInvestigation.md | 79 +-
.../Threat_Investigation_master.ipynb | 233 +-
spot-oa/oa/flow/README.md | 134 +-
spot-oa/oa/flow/__init__.py | 16 +
spot-oa/oa/flow/flow_conf.json | 27 +-
.../ipynb_templates/Advanced_Mode_master.ipynb | 292 +
spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md | 97 -
.../Edge_Investigation_master.ipynb | 407 -
.../flow/ipynb_templates/ThreatInvestigation.md | 191 +-
.../Threat_Investigation_master.ipynb | 681 +-
spot-oa/oa/proxy/README.md | 139 +-
spot-oa/oa/proxy/__init__.py | 16 +
.../ipynb_templates/Advanced_Mode_master.ipynb | 296 +
.../oa/proxy/ipynb_templates/EdgeNotebook.md | 74 -
.../Edge_Investigation_master.ipynb | 229 -
.../ipynb_templates/ThreatInvestigation.md | 88 +-
.../Threat_Investigation_master.ipynb | 336 +-
spot-oa/oa/proxy/proxy_conf.json | 10 +-
spot-oa/oa/start_oa.py | 31 +-
spot-oa/oa/utils.py | 12 +
spot-oa/requirements.txt | 18 +-
spot-oa/runIpython.sh | 11 +-
spot-oa/ui/README.md | 27 +-
spot-oa/ui/css/bootstrap-spot.min.css | 16 +
spot-oa/ui/css/dendrogram.css | 16 +
spot-oa/ui/css/main.css | 71 +
spot-oa/ui/dns/ipython_notebook.html | 108 +
.../components/DetailsDendrogramPanel.react.js | 22 +-
.../ui/dns/js/components/DetailsPanel.react.js | 17 +-
.../js/components/DetailsTablePanel.react.js | 21 +-
.../ui/dns/js/components/FilterInput.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 21 +-
.../dns/js/components/NetworkViewPanel.react.js | 19 +-
.../ui/dns/js/components/ScoreNotebook.react.js | 206 +
.../dns/js/components/SuspiciousPanel.react.js | 39 +-
spot-oa/ui/dns/js/constants/DnsConstants.js | 12 -
spot-oa/ui/dns/js/notebooks.js | 72 +
spot-oa/ui/dns/js/stores/CommentsStore.js | 86 +-
spot-oa/ui/dns/js/stores/DendrogramStore.js | 103 +-
spot-oa/ui/dns/js/stores/DetailsStore.js | 129 +-
.../dns/js/stores/IncidentProgressionStore.js | 154 +-
spot-oa/ui/dns/js/stores/NotebookStore.js | 107 +
spot-oa/ui/dns/js/stores/SuspiciousStore.js | 352 +-
spot-oa/ui/dns/js/storyboard.js | 23 +-
spot-oa/ui/dns/js/suspicious.js | 51 +-
spot-oa/ui/dns/js/threat-investigation.js | 23 +-
spot-oa/ui/dns/package.json | 6 +-
spot-oa/ui/dns/storyboard.html | 58 +-
spot-oa/ui/dns/suspicious.html | 61 +-
spot-oa/ui/dns/threat-investigation.html | 58 +-
spot-oa/ui/flow/ipython_notebook.html | 108 +
.../js/components/DetailsChordsPanel.react.js | 17 +-
.../ui/flow/js/components/DetailsPanel.react.js | 17 +-
.../js/components/DetailsTablePanel.react.js | 17 +-
.../ui/flow/js/components/FilterInput.react.js | 17 +-
.../flow/js/components/GlobeViewPanel.react.js | 17 +-
.../js/components/ImpactAnalysisPanel.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 17 +-
.../js/components/NetworkViewPanel.react.js | 20 +-
.../flow/js/components/ScoreNotebook.react.js | 221 +
.../flow/js/components/SuspiciousPanel.react.js | 20 +-
.../flow/js/components/TimelinePanel.react.js | 17 +-
.../ui/flow/js/constants/NetflowConstants.js | 27 +-
spot-oa/ui/flow/js/lib/topojson.min.js | 2 -
spot-oa/ui/flow/js/notebooks.js | 72 +
spot-oa/ui/flow/js/stores/ChordsDiagramStore.js | 106 +-
spot-oa/ui/flow/js/stores/CommentsStore.js | 77 +-
spot-oa/ui/flow/js/stores/DetailsStore.js | 159 +-
spot-oa/ui/flow/js/stores/GlobeViewStore.js | 119 +-
.../ui/flow/js/stores/ImpactAnalysisStore.js | 97 +-
.../flow/js/stores/IncidentProgressionStore.js | 87 +-
spot-oa/ui/flow/js/stores/NotebookStore.js | 107 +
spot-oa/ui/flow/js/stores/SuspiciousStore.js | 306 +-
spot-oa/ui/flow/js/stores/TimelineStore.js | 99 +-
spot-oa/ui/flow/js/storyboard.js | 23 +-
spot-oa/ui/flow/js/suspicious.js | 51 +-
spot-oa/ui/flow/js/threat-investigation.js | 23 +-
spot-oa/ui/flow/package.json | 6 +-
spot-oa/ui/flow/storyboard.html | 58 +-
spot-oa/ui/flow/suspicious.html | 62 +-
spot-oa/ui/flow/threat-investigation.html | 58 +-
spot-oa/ui/images/favicon/browserconfig.xml | 14 +
spot-oa/ui/ingest-summary.html | 183 -
spot-oa/ui/ingest/ingest-summary.html | 132 +
spot-oa/ui/ipython/custom/ajax-loader.gif | Bin 723 -> 0 bytes
spot-oa/ui/ipython/custom/custom.css | 145 -
spot-oa/ui/ipython/custom/custom.js | 337 -
spot-oa/ui/js/actions/EdInActions.js | 34 +-
spot-oa/ui/js/actions/InSumActions.js | 17 +-
spot-oa/ui/js/actions/SpotActions.js | 23 +-
spot-oa/ui/js/actions/StoryboardActions.js | 17 +-
.../ui/js/actions/__tests__/EdInActions.test.js | 15 +-
.../ui/js/actions/__tests__/SpotActions.test.js | 16 +-
.../actions/__tests__/StoryboardActions.test.js | 16 +-
spot-oa/ui/js/components/ChartMixin.react.js | 17 +-
.../js/components/ContentLoaderMixin.react.js | 17 +-
spot-oa/ui/js/components/DateInput.react.js | 17 +-
.../ui/js/components/DendrogramMixin.react.js | 17 +-
.../ui/js/components/DetailsGridMixin.react.js | 18 +-
.../ExecutiveThreatBriefingPanel.react.js | 17 +-
.../ui/js/components/GridPanelMixin.react.js | 17 +-
.../js/components/IPythonNotebookPanel.react.js | 25 +-
.../js/components/IngestSummaryPanel.react.js | 19 +-
spot-oa/ui/js/components/OptionPicker.react.js | 17 +-
spot-oa/ui/js/components/Panel.react.js | 28 +-
spot-oa/ui/js/components/PanelRow.react.js | 20 +-
.../components/PolloNetworkViewMixin.react.js | 19 +-
.../js/components/SuspiciousGridMixin.react.js | 17 +-
spot-oa/ui/js/components/TimelineMixin.react.js | 17 +-
.../components/scoring/ButtonsScoring.react.js | 48 +
.../scoring/FilterSelectInput.react.js | 45 +
.../js/components/scoring/RatingInput.react.js | 42 +
.../js/components/scoring/ScoreMessage.react.js | 38 +
.../scoring/SearchGlobalInput.react.js | 49 +
.../js/components/scoring/SelectInput.react.js | 46 +
spot-oa/ui/js/constants/SpotConstants.js | 26 +-
spot-oa/ui/js/cubehelix/cubehelix.js | 17 +-
spot-oa/ui/js/dispatchers/SpotDispatcher.js | 17 +-
spot-oa/ui/js/ingest-summary.js | 35 +-
spot-oa/ui/js/menu/components/MainMenu.react.js | 73 +
spot-oa/ui/js/menu/menu.js | 52 +
spot-oa/ui/js/menu/stores/MainMenuStore.js | 57 +
spot-oa/ui/js/stores/GraphQLStore.js | 89 +
spot-oa/ui/js/stores/IngestSummaryStore.js | 236 +-
spot-oa/ui/js/stores/JsonStore.js | 17 +-
spot-oa/ui/js/stores/ObservableGraphQLStore.js | 55 +
.../stores/ObservableWithHeadersGraphQLStore.js | 31 +
spot-oa/ui/js/stores/RestStore.js | 17 +-
spot-oa/ui/js/stores/SpotStore.js | 17 +-
.../ui/js/stores/__tests__/JsonStore.test.js | 16 +-
.../ui/js/stores/__tests__/RestStore.test.js | 16 +-
.../ui/js/stores/__tests__/SpotStore.test.js | 16 +-
spot-oa/ui/js/tooltip.js | 17 +-
spot-oa/ui/js/utils/CategoryLayout.js | 17 +-
spot-oa/ui/js/utils/DateUtils.js | 17 +-
spot-oa/ui/js/utils/SpotUtils.js | 52 +-
.../js/utils/__tests__/CategoryLayout.test.js | 16 +-
spot-oa/ui/js/utils/__tests__/DateUtils.test.js | 16 +-
spot-oa/ui/js/utils/__tests__/SpotUtils.test.js | 16 +-
spot-oa/ui/package.json | 5 +-
spot-oa/ui/proxy/ipython_notebook.html | 108 +
.../proxy/js/components/DetailsPanel.react.js | 20 +-
.../ui/proxy/js/components/FilterInput.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 17 +-
.../js/components/NetworkViewPanel.react.js | 23 +-
.../proxy/js/components/ScoreNotebook.react.js | 199 +
.../js/components/SuspiciousPanel.react.js | 20 +-
.../proxy/js/components/TimelinePanel.react.js | 18 +-
spot-oa/ui/proxy/js/constants/ProxyConstants.js | 12 -
spot-oa/ui/proxy/js/notebooks.js | 72 +
spot-oa/ui/proxy/js/stores/CommentsStore.js | 77 +-
spot-oa/ui/proxy/js/stores/DetailsStore.js | 140 +-
.../proxy/js/stores/IncidentProgressionStore.js | 83 +-
spot-oa/ui/proxy/js/stores/NotebookStore.js | 107 +
spot-oa/ui/proxy/js/stores/SuspiciousStore.js | 310 +-
spot-oa/ui/proxy/js/stores/TimelineStore.js | 108 +-
spot-oa/ui/proxy/js/storyboard.js | 23 +-
spot-oa/ui/proxy/js/suspicious.js | 45 +-
spot-oa/ui/proxy/js/threat-investigation.js | 23 +-
spot-oa/ui/proxy/package.json | 6 +-
spot-oa/ui/proxy/storyboard.html | 58 +-
spot-oa/ui/proxy/suspicious.html | 61 +-
spot-oa/ui/proxy/threat-investigation.html | 58 +-
spot-setup/README.md | 63 +-
spot-setup/create_dns_avro_parquet.hql | 37 -
spot-setup/create_dns_parquet.hql | 163 +
spot-setup/create_flow_avro_parquet.hql | 72 -
spot-setup/create_flow_parquet.hql | 195 +
spot-setup/create_proxy_avro_parquet.hql | 76 -
spot-setup/create_proxy_parquet.hql | 177 +
spot-setup/hdfs_setup.sh | 47 +-
spot-setup/migrate_to_spot_1_0.py | 102 +
spot-setup/migration/README.md | 71 +
spot-setup/migration/__init__.py | 16 +
.../migration/create_dns_migration_tables.hql | 113 +
.../migration/create_flow_migration_tables.hql | 126 +
.../migration/create_proxy_migration_tables.hql | 127 +
.../migration/drop_dns_migration_tables.hql | 23 +
.../migration/drop_flow_migration_tables.hql | 23 +
.../migration/drop_proxy_migration_tables.hql | 23 +
spot-setup/migration/migrate_old_dns_data.py | 248 +
spot-setup/migration/migrate_old_flow_data.py | 240 +
spot-setup/migration/migrate_old_proxy_data.py | 236 +
spot-setup/migration/spot_conf_migration.py | 135 +
spot-setup/migration/utilities.py | 89 +
spot-setup/odm/README.md | 68 +
spot-setup/odm/create_endpoint_context_avro.sql | 58 +
spot-setup/odm/create_endpoint_context_pqt.sql | 57 +
spot-setup/odm/create_event_avro.sql | 302 +
spot-setup/odm/create_event_pqt.sql | 301 +
spot-setup/odm/create_network_context_avro.sql | 48 +
spot-setup/odm/create_network_context_pqt.sql | 47 +
.../create_threat_intelligence_context_avro.sql | 76 +
.../create_threat_intelligence_context_pqt.sql | 75 +
spot-setup/odm/create_user_context_avro.sql | 51 +
spot-setup/odm/create_user_context_pqt.sql | 50 +
.../odm/create_vulnerability_context_avro.sql | 32 +
.../odm/create_vulnerability_context_pqt.sql | 31 +
spot-setup/odm/endpoint_context.avsc | 44 +
spot-setup/odm/event.avsc | 266 +
spot-setup/odm/network_context.avsc | 34 +
spot-setup/odm/odm_setup.sh | 105 +
spot-setup/odm/threat_intelligence_context.avsc | 62 +
spot-setup/odm/user_context.avsc | 37 +
spot-setup/odm/vulnerability_context.avsc | 18 +
spot-setup/spot.conf | 41 +-
329 files changed, 28898 insertions(+), 1006297 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/6aa2c66d/spot-ml/ml_ops.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/6aa2c66d/spot-setup/spot.conf
----------------------------------------------------------------------
diff --cc spot-setup/spot.conf
index abe8030,50bd851..03f221a
--- a/spot-setup/spot.conf
+++ b/spot-setup/spot.conf
@@@ -11,19 -30,9 +30,13 @@@ PROXY_PATH=${HUSER}/${DSOURCE}/hive/y=$
FLOW_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
HPATH=${HUSER}/${DSOURCE}/scored_results/${FDATE}
+FLOW_TABLE=flow_view
+DNS_TABLE=dns_view
+PROXY_TABLE=proxy_view
+
#impala config
- IMPALA_DEM='node04'
-
- #kerberos config
- KRB_AUTH=false
- KINITPATH=
- KINITOPTS=
- KEYTABPATH=
- KRB_USER=
+ IMPALA_DEM=node04
+ IMPALA_PORT=21050
#local fs base user and data source config
LUSER='/home/spot'