You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 16:31:37 UTC
[1/2] incubator-spot git commit: SPOT-233 closes
apache/incubator-spot#119
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-181_ODM 016a5e4c9 -> 0ee2d06b8
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/threat_intelligence_context.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/threat_intelligence_context.avsc b/spot-setup/odm/threat_intelligence_context.avsc
new file mode 100644
index 0000000..11a3985
--- /dev/null
+++ b/spot-setup/odm/threat_intelligence_context.avsc
@@ -0,0 +1,62 @@
+{
+ "namespace":"org.apache.spot",
+ "name":"threat_intelligence_context",
+ "type": "record",
+ "fields": [
+ {"name":"ti_source","type":["null","string"],"doc":"TI Provider, Open Source List, Internally Developed, LE Tip, Other", "default": null},
+ {"name":"ti_provider_id", "type":["null","string"],"doc":"Anomali, CrowdStrike, Mandiant, Alienvault OTX, USCERT, etc", "default": null},
+ {"name":"ti_indicator_id", "type":["null","string"],"doc":"Unique IQ from the provider", "default": null},
+ {"name":"ti_indicator_desc", "type":["null","string"],"doc":"Full Text descriptor and links of the Indicator and associated information", "default": null},
+ {"name":"ti_date_added", "type":["null","long"],"doc":"Date first added by the provider", "default": null},
+ {"name":"ti_date_modified", "type":["null","long"],"doc":"Date last updated by the provider.", "default": null},
+ {"name":"ti_risk_impact", "type":["null","string"],"doc":"Likely Targets what function within the organization?", "default": null},
+ {"name":"ti_severity", "type":["null","string"],"doc":"Nation State, Targeted, Advanced, Commodity, Other", "default": null},
+ {"name":"ti_category", "type":["null","string"],"doc":"Ecrime, Hacktivism, Geo Pollitical, Foreign Intelligence Service", "default": null},
+ {"name":"ti_campaign_name", "type":["null","string"],"doc":"Internal Campaign designation", "default": null},
+ {"name":"ti_deployed_location", "type":["null",{"type":"array", "items":"string"}],"doc":"Where this indicator should be matched for applicability (Core, Perimeter, Network, Endpoint, Logs, ALL, etc)", "default": null},
+ {"name":"ti_associated_incidents", "type":["null","string"],"doc":"Known Associated Incident ID's", "default": null},
+ {"name":"ti_adversarial_identification_group", "type":["null","string"],"doc":"Adversary Group designation usually provided by the provider.", "default": null},
+ {"name":"ti_adversarial_identification_tactics", "type":["null","string"],"doc":"Known Adversary Tactics as indicated by the source provider.", "default": null},
+ {"name":"ti_adversarial_identification_reports", "type":["null","string"],"doc":"Linked Adversary reports."},
+ {"name":"ti_phase", "type":["null","string"],"doc":"Discovery, Weaponization, Delivery, C2, Exploitation, Actions on Objectives, etc", "default": null},
+ {"name":"ti_indicator_cve", "type":["null","string"],"doc":"MITRE CVE Link(s)", "default": null},
+ {"name":"ti_indicator_ip4", "type":["null",{"type":"array", "items":"long"}],"doc":"CIDR noted IPv4 Address Indicated by Threat Intelligence", "default": null},
+ {"name":"ti_indicator_ip4_str", "type":["null",{"type":"array", "items":"string"}],"doc":"CIDR noted IPv4 Address Indicated by Threat Intelligence", "default": null},
+ {"name":"ti_indicator_ip6", "type":["null",{"type":"array", "items":"long"}],"doc":"IPv6 Address Indicated by Threat Intelligence", "default": null},
+ {"name":"ti_indicator_ip6_str", "type":["null",{"type":"array", "items":"string"}],"doc":"IPv6 Address Indicated by Threat Intelligence", "default": null},
+ {"name":"ti_indicator_domain", "type":["null","string"],"doc":"Domain Name(s)", "default": null},
+ {"name":"ti_indicator_hostname", "type":["null","string"],"doc":"Host or Subdomain Name(es)", "default": null},
+ {"name":"ti_indicator_email", "type":["null",{"type":"array", "items":"string"}],"doc":"Email addresses associated with Indicator", "default": null},
+ {"name":"ti_indicator_url", "type":["null",{"type":"array", "items":"string"}],"doc":"URL(s) associated with indicatorv", "default": null},
+ {"name":"ti_indicator_uri", "type":["null",{"type":"array", "items":"string"}],"doc":"URI(s) associated with indicator", "default": null},
+ {"name":"ti_indicator_file_hash", "type":["null","string"],"doc":"File Hash Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_file_path", "type":["null","string"],"doc":"File Path Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_mutex", "type":["null","string"],"doc":"MUTEX Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_md5", "type":["null","string"],"doc":"MD5 Hash Sum Value", "default": null},
+ {"name":"ti_indicator_sha1", "type":["null","string"],"doc":"SHA1 Hash Sum Value", "default": null},
+ {"name":"ti_indicator_sha256", "type":["null","string"],"doc":"SHA256 Hash Sum Value", "default": null},
+ {"name":"ti_indicator_device_path", "type":["null","string"],"doc":"Device Path Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_drive", "type":["null","string"],"doc":"Drive Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_file_name", "type":["null","string"],"doc":"File Name Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_file_extension", "type":["null","string"],"doc":"File Extension Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_file_size", "type":["null","string"],"doc":"File Size Value associated with the indicator.", "default": null},
+ {"name":"ti_indicator_file_created", "type":["null","long"],"doc":"Date File value associated with the indicator was created.", "default": null},
+ {"name":"ti_indicator_file_accessed", "type":["null","long"],"doc":"Date File value associated with the indicator was last accessed.", "default": null},
+ {"name":"ti_indicator_file_changed", "type":["null","long"],"doc":"Date File value associated with the indicator was last changed.", "default": null},
+ {"name":"ti_indicator_file_entropy", "type":["null","string"],"doc":"Calculated entropy value associated with the file indicated.", "default": null},
+ {"name":"ti_indicator_file_attributes", "type":["null",{"type":"array", "items":"string"}],"doc":"Read Only, System, Hidden, Directory, Archive, Device, Temporary, SparseFile, Compressed, Encrypted, Index, Deleted, etc", "default": null},
+ {"name":"ti_indicator_user_name", "type":["null","string"],"doc":"username associated with the indicator.", "default": null},
+ {"name":"ti_indicator_security_id", "type":["null","string"],"doc":"if known securityID associated with the indicator.", "default": null},
+ {"name":"ti_indicator_pe_info", "type":["null",{"type":"array", "items":"string"}],"doc":"Subsystem, BaseAddress, PETImeStamp, Expert, JumpCodes, DetectedAnomalies, DigitalSignatures,VersionInfo, ResourceInfo,Imported Modules", "default": null},
+ {"name":"ti_indicator_pe_type", "type":["null",{"type":"array", "items":"string"}],"doc":"Executable, DLL, Invalid, Unknown, Native, Windows_GUI, OS2, POSIX, EFI, etc", "default": null},
+ {"name":"ti_indicator_strings", "type":["null",{"type":"array", "items":"string"}],"doc":"Any strings associated with the file indicated that might be useful in identification or further indicator development or adversary identification.", "default": null},
+ {"name":"ti_indicator_org", "type":["null","string"],"doc":"Name of the business that owns the IP address associated with the indicator", "default": null},
+ {"name":"ti_indicator_reg_name", "type":["null","string"],"doc":"Name of the person who registered the domain", "default": null},
+ {"name":"ti_indicator_reg_email", "type":["null","string"],"doc":"Email address of the person who registered the domain", "default": null},
+ {"name":"ti_indicator_reg_org", "type":["null","string"],"doc":"Name of the organisation that registered the domain", "default": null},
+ {"name":"ti_indicator_reg_phone", "type":["null","string"],"doc":"Phone number associated with the domain registered", "default": null},
+ {"name":"ti_tags", "type":["null","string"],"doc":"Additional comments/associations from the feed", "default": null},
+ {"name":"ti_threat_type", "type":["null","string"],"doc":"malware, compromised, apt, c2, etc...", "default": null}
+ ],
+ "doc": "A view schema for storing Apache Spot Threat Intelligence Context data."
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/user_context.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/user_context.avsc b/spot-setup/odm/user_context.avsc
new file mode 100644
index 0000000..9c9f7e5
--- /dev/null
+++ b/spot-setup/odm/user_context.avsc
@@ -0,0 +1,37 @@
+{
+ "namespace":"org.apache.spot",
+ "name":"user_context",
+ "type": "record",
+ "fields": [
+ {"name":"dvc_time","type":["null","long"],"doc":"Timestamp from when the user context information is obtained","default":null},
+ {"name":"user_created", "type":["null","long"],"doc":"Timestamp from when user was created","default":null},
+ {"name":"user_changed", "type":["null","long"],"doc":"Timestamp from when user was updated","default":null},
+ {"name":"user_last_logon", "type":["null","long"],"doc":"Timestamp from when user last logged on","default":null},
+ {"name":"user_logon_count", "type":["null","int"],"doc":"Number of times account has logged on","default":null},
+ {"name":"user_last_reset", "type":["null","long"],"doc":"Timestamp from when user last reset password","default":null},
+ {"name":"user_expiration", "type":["null","long"],"doc":"Date/time when user expires","default":null},
+ {"name":"user_image", "type":["null","binary"],"doc":"Image data for user","default":null},
+ {"name":"user_id", "type":["null","string"],"doc":"Unique user id","default":null},
+ {"name":"user_name", "type":["null","string"],"doc":"Username in event log/alert","default":null},
+ {"name":"user_name_first", "type":["null","string"],"doc":"First name","default":null},
+ {"name":"user_name_middle", "type":["null","string"],"doc":"Middle name","default":null},
+ {"name":"user_name_last", "type":["null","string"],"doc":"Last name","default":null},
+ {"name":"user_name_mgr", "type":["null","string"],"doc":"Manager’s name","default":null},
+ {"name":"user_phone", "type":["null","string"],"doc":"Phone number","default":null},
+ {"name":"user_email", "type":["null","string"],"doc":"Email address","default":null},
+ {"name":"user_code", "type":["null","string"],"doc":"Job code","default":null},
+ {"name":"user_loc", "type":["null","string"],"doc":"Location","default":null},
+ {"name":"user_departm", "type":["null","string"],"doc":"Department","default":null},
+ {"name":"user_dn", "type":["null","string"],"doc":"Distinguished name","default":null},
+ {"name":"user_ou", "type":["null","string"],"doc":"Organizational unit","default":null},
+ {"name":"user_empid", "type":["null","string"],"doc":"Employee ID","default":null},
+ {"name":"user_title", "type":["null","string"],"doc":"Job Title","default":null},
+ {"name":"user_groups", "type":["null",{"type":"array", "items":"string"}],"doc":"Groups to which the user belongs","default":null},
+ {"name":"dvc_type", "type":["null","string"],"doc":"Device type that generated the user context data","default":null},
+ {"name":"dvc_vendor", "type":["null","string"],"doc":"Vendor","default":null},
+ {"name":"user_risk", "type":["null","float"],"doc":"Risk score","default":null},
+ {"name":"dvc_version", "type":["null","string"],"doc":"Version","default":null},
+ {"name":"additional_attrs", "type":["null",{"type":"map","values":["null","string"]}],"default":null,"doc":"Additional attributes of user"}
+ ],
+ "doc": "A view schema for storing Apache Spot User Context data."
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/vulnerability_context.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/vulnerability_context.avsc b/spot-setup/odm/vulnerability_context.avsc
new file mode 100644
index 0000000..933b8d1
--- /dev/null
+++ b/spot-setup/odm/vulnerability_context.avsc
@@ -0,0 +1,18 @@
+{
+ "namespace":"org.apache.spot",
+ "name":"vulnerability_context",
+ "type": "record",
+ "fields": [
+ {"name":"vuln_id","type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_title", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_description", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_solution", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_type", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_category", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_severity", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_created", "type":["null","long"],"doc":"TBD", "default": null},
+ {"name":"vuln_updated", "type":["null","long"],"doc":"TBD", "default": null},
+ {"name":"additional_attrs", "type":["null",{"type":"map","values":["null","string"]}],"default":null,"doc":"Additional attributes of vulnerability"}
+ ],
+ "doc": "A view schema for storing Apache Spot Vulnerability Context data."
+ }
\ No newline at end of file
[2/2] incubator-spot git commit: SPOT-233 closes
apache/incubator-spot#119
Posted by na...@apache.org.
SPOT-233 closes apache/incubator-spot#119
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0ee2d06b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0ee2d06b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0ee2d06b
Branch: refs/heads/SPOT-181_ODM
Commit: 0ee2d06b8ceca01c321251e4ae32fd17026b228c
Parents: 016a5e4
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Tue Sep 26 09:29:42 2017 -0700
Committer: natedogs911 <na...@gmail.com>
Committed: Tue Sep 26 09:29:42 2017 -0700
----------------------------------------------------------------------
spot-setup/README.md | 4 +-
spot-setup/odm/README.md | 68 +++++
spot-setup/odm/create_endpoint_context_avro.sql | 58 ++++
spot-setup/odm/create_endpoint_context_pqt.sql | 57 ++++
spot-setup/odm/create_event_avro.sql | 302 +++++++++++++++++++
spot-setup/odm/create_event_pqt.sql | 301 ++++++++++++++++++
spot-setup/odm/create_network_context_avro.sql | 48 +++
spot-setup/odm/create_network_context_pqt.sql | 47 +++
.../create_threat_intelligence_context_avro.sql | 76 +++++
.../create_threat_intelligence_context_pqt.sql | 75 +++++
spot-setup/odm/create_user_context_avro.sql | 51 ++++
spot-setup/odm/create_user_context_pqt.sql | 50 +++
.../odm/create_vulnerability_context_avro.sql | 32 ++
.../odm/create_vulnerability_context_pqt.sql | 31 ++
spot-setup/odm/endpoint_context.avsc | 44 +++
spot-setup/odm/event.avsc | 266 ++++++++++++++++
spot-setup/odm/network_context.avsc | 34 +++
spot-setup/odm/odm_setup.sh | 105 +++++++
spot-setup/odm/threat_intelligence_context.avsc | 62 ++++
spot-setup/odm/user_context.avsc | 37 +++
spot-setup/odm/vulnerability_context.avsc | 18 ++
21 files changed, 1764 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/README.md b/spot-setup/README.md
index 1ac02f2..4a80c1b 100644
--- a/spot-setup/README.md
+++ b/spot-setup/README.md
@@ -26,7 +26,7 @@ The main script in the repository is **hdfs_setup.sh** which is responsible of l
This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, Kerberos information and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
-To read more about these variables, please review the [wiki] (https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration).
+To read more about these variables, please review the [wiki] (http://spot.incubator.apache.org/doc/#configuration).
## Database Query Scripts
@@ -61,7 +61,7 @@ Create a pull request and contact the maintainers.
## Issues
-Report issues at the Apache Spot [issues] (https://github.com/Open-Network-Insight/open-network-insight/issues) page.
+Report issues at the Apache Spot [issues] (https://issues.apache.org/jira/projects/SPOT/issues) page.
## Maintainers
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/odm/README.md b/spot-setup/odm/README.md
new file mode 100644
index 0000000..bc5bbb7
--- /dev/null
+++ b/spot-setup/odm/README.md
@@ -0,0 +1,68 @@
+# spot-setup/odm - Open Data Model (ODM) Setup
+
+## Intended Audience
+
+This document is intended for any developer or sysadmin in learning the technical aspects or in contributing to the setup installation process of the Apache Spot solution.
+
+## Getting Started
+
+This information will help you to get started on contributing to the ODM component of the Apache Spot Setup repository.
+For information about installing and running Apache Spot go to our [Installation Guide](http://spot.apache.org/doc/).
+spot-setup/odm contains the scripts to setup HDFS for the Apache Spot Open Data Model (ODM).
+It will create the folder and database structure needed to run Apache Spot on HDFS and HIVE respectively.
+spot-setup/odm is a component of Apache Spot and is executed in the initial configuration after Linux user creation and before Ingest installation.
+
+## Prerequisites
+
+To collaborate and run spot-setup, it is required the following prerequisites:
+- A running Hadoop cluster
+- Linux user account created in all nodes with sudo privileges
+- HDFS ACLs must be enabled (dfs.namenode.acls.enabled=true)
+
+## General Description
+
+The main script in the repository is **odm_setup.sh** which is responsible of loading environment variables,
+creating folders in Hadoop for the ODM data (event, user_context, endpoint_context, network_context, threat_intelligence_context, vulnerability_context),
+create the Hive database,
+and finally execute hive query scripts that creates Hive tables needed to access Apache Spot data through the ODM.
+
+## Environment Variables
+
+**spot.conf** is the file storing the variables needed during the installation process including node assignment, User interface, Machine Learning and Ingest gateway nodes.
+This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, Kerberos information and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
+To read more about these variables, please review the [wiki] (https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration).
+
+By default, **odm_setup.sh** expects **spot.conf** to be located in the **/etc** directory on the node. An example spot.conf file is provided in the spot-setup parent directory to help you get started.
+
+## ODM Database Query Scripts
+
+spot-setup/odm contains a script per each table as specified in the Apache Spot Open Data Model (ODM) document.
+These HQL scripts are intended to be executed as a Hive statement and must comply HQL standards.
+
+We want to create tables in Avro/Parquet format to get a faster query performance. This format is an industry standard and you can find more information about it on:
+- Avro is a data serialization system - https://avro.apache.org/
+- Parquet is a columnar storage format - https://parquet.apache.org/
+
+The ODM database query scripts are referenced and executed as part of running the **odm_setup.sh** script.
+Data is meant to be ingested directly into the ODM directories that are tied to the external tables created from the script.
+
+#### ODM Tables
+- event - includes event logs from common data sources used to detect threats and includes network flows, operating system logs, IPS/IDS logs, firewall logs, proxy logs, web logs, DLP logs, etc.
+- user_context - includes information from user and identity management systems.
+- endpoint_context - includes information about endpoint systems (servers, workstations, routers, switches, etc.) and can be sourced from asset management systems, vulnerability scanners, and endpoint management/detection/response systems
+- threat_intelligence_context - includes contextual information about URLs, domains, websites, files and others.
+- network_context - includes information about the network, which can be gleaned from Whois servers, asset databases and other similar data sources.
+- vulnerabilty_context - includes information about vulnerabilities present on endpoint systems that can be sourced from vulnerability scanners and management systems.
+
+## Licensing
+
+spot-setup/odm is licensed under Apache Version 2.0
+
+## Contributing
+
+Create a pull request and contact the maintainers.
+
+## Issues
+
+Report issues at the Apache Spot [issues] (https://github.com/Open-Network-Insight/open-network-insight/issues) page.
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_endpoint_context_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_endpoint_context_avro.sql b/spot-setup/odm/create_endpoint_context_avro.sql
new file mode 100644
index 0000000..1428123
--- /dev/null
+++ b/spot-setup/odm/create_endpoint_context_avro.sql
@@ -0,0 +1,58 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+dvc_time bigint,
+end_ip4 bigint,
+end_ip4_str string,
+end_ip6 bigint,
+end_ip6_str string,
+end_os string,
+end_os_version string,
+end_os_sp string,
+end_tz string,
+end_hotfixes array<string>,
+end_disks array<string>,
+end_removeables array<string>,
+end_nics array<string>,
+end_drivers array<string>,
+end_users array<string>,
+end_host string,
+end_mac string,
+end_owner string,
+end_vulns array<string>,
+end_loc string,
+end_departm string,
+end_company string,
+end_regs array<string>,
+end_svcs array<string>,
+end_procs array<string>,
+end_criticality string,
+end_apps array<string>,
+end_desc string,
+dvc_type string,
+dvc_vendor string,
+dvc_version string,
+end_architecture string,
+end_uuid string,
+end_risk float,
+end_memtotal int,
+additional_attrs map<string,string>)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}')
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_endpoint_context_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_endpoint_context_pqt.sql b/spot-setup/odm/create_endpoint_context_pqt.sql
new file mode 100644
index 0000000..e460e6a
--- /dev/null
+++ b/spot-setup/odm/create_endpoint_context_pqt.sql
@@ -0,0 +1,57 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+dvc_time bigint,
+end_ip4 bigint,
+end_ip4_str string,
+end_ip6 bigint,
+end_ip6_str string,
+end_os string,
+end_os_version string,
+end_os_sp string,
+end_tz string,
+end_hotfixes array<string>,
+end_disks array<string>,
+end_removeables array<string>,
+end_nics array<string>,
+end_drivers array<string>,
+end_users array<string>,
+end_host string,
+end_mac string,
+end_owner string,
+end_vulns array<string>,
+end_loc string,
+end_departm string,
+end_company string,
+end_regs array<string>,
+end_svcs array<string>,
+end_procs array<string>,
+end_criticality string,
+end_apps array<string>,
+end_desc string,
+dvc_type string,
+dvc_vendor string,
+dvc_version string,
+end_architecture string,
+end_uuid string,
+end_risk float,
+end_memtotal int,
+additional_attrs map<string,string>)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}'
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_event_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_event_avro.sql b/spot-setup/odm/create_event_avro.sql
new file mode 100644
index 0000000..16945cf
--- /dev/null
+++ b/spot-setup/odm/create_event_avro.sql
@@ -0,0 +1,302 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+-- Common
+event_time bigint,
+begintime bigint,
+endtime bigint,
+event_insertime bigint,
+lastupdatetime bigint,
+duration float,
+event_id string,
+name string,
+org string,
+type string,
+n_proto string,
+a_proto string,
+msg string,
+mac string,
+severity string,
+raw string,
+risk float,
+code string,
+category string,
+query string,
+service string,
+state string,
+in_bytes int,
+out_bytes int,
+xref string,
+version string,
+api string,
+parameter string,
+action string,
+proc string,
+app string,
+disposition string,
+prevalence string,
+confidence string,
+sensitivity string,
+count int,
+company string,
+additional_attrs map<string,string>,
+totrust string,
+fromtrust string,
+rule string,
+threat string,
+pcap_id int,
+-- Device
+dvc_time bigint,
+dvc_ip4 bigint,
+dvc_ip4_str string,
+dvc_ip6 bigint,
+dvc_ip6_str string,
+dvc_host string,
+dvc_domain string,
+dvc_type string,
+dvc_vendor string,
+dvc_fwd_ip4 bigint,
+dvc_fwd_ip4_str string,
+dvc_fwd_ip6 bigint,
+dvc_fwd_ip6_str string,
+dvc_version string,
+-- Network
+src_ip4 bigint,
+src_ip4_str string,
+src_ip6 bigint,
+src_ip6_str string,
+src_host string,
+src_domain string,
+src_port int,
+src_country_code string,
+src_country_name string,
+src_region string,
+src_city string,
+src_lat int,
+src_long int,
+dst_ip4 bigint,
+dst_ip4_str string,
+dst_ip6 bigint,
+dst_ip6_str string,
+dst_host string,
+dst_domain string,
+dst_port int,
+dst_country_code string,
+dst_country_name string,
+dst_region string,
+dst_city string,
+dst_lat int,
+dst_long int,
+src_asn int,
+dst_asn int,
+net_direction string,
+net_flags string,
+-- File
+file_name string,
+file_path string,
+file_atime bigint,
+file_acls string,
+file_type string,
+file_size int,
+file_desc string,
+file_hash string,
+file_hash_type string,
+-- Endpoint
+end_object string,
+end_action string,
+end_msg string,
+end_app string,
+end_location string,
+end_proc string,
+-- User
+user_name string,
+src_user_name string,
+dst_user_name string,
+user_email string,
+user_id string,
+user_loc string,
+user_desc string,
+-- DNS
+dns_class string,
+dns_len int,
+dns_query string,
+dns_response_code string,
+dns_answers string,
+dns_type int,
+-- Proxy
+prx_category string,
+prx_browser string,
+prx_code string,
+prx_referrer string,
+prx_host string,
+prx_filter_rule string,
+prx_filter_result string,
+prx_query string,
+prx_action string,
+prx_method string,
+prx_type string,
+-- HTTP
+http_request_method string,
+http_request_uri string,
+http_request_body_len int,
+http_request_user_name string,
+http_request_password string,
+http_request_proxied string,
+http_request_headers map<string,string>,
+http_response_status_code int,
+http_response_status_msg string,
+http_response_body_len int,
+http_response_info_code int,
+http_response_info_msg string,
+http_response_resp_fuids string,
+http_response_mime_types string,
+http_response_headers map<string,string>,
+-- SMTP
+smtp_trans_depth int,
+smtp_headers_helo string,
+smtp_headers_mailfrom string,
+smtp_headers_rcptto string,
+smtp_headers_date string,
+smtp_headers_from string,
+smtp_headers_to string,
+smtp_headers_reply_to string,
+smtp_headers_msg_id string,
+smtp_headers_in_reply_to string,
+smtp_headers_subject string,
+smtp_headers_x_originating_ip4 bigint,
+smtp_headers_x_originating_ip4_str string,
+smtp_headers_first_received string,
+smtp_headers_second_received string,
+smtp_last_reply string,
+smtp_path string,
+smtp_user_agent string,
+smtp_tls boolean,
+smtp_is_webmail boolean,
+-- FTP
+ftp_user_name string,
+ftp_password string,
+ftp_command string,
+ftp_arg string,
+ftp_mime_type string,
+ftp_file_size int,
+ftp_reply_code int,
+ftp_reply_msg string,
+ftp_data_channel_passive boolean,
+ftp_data_channel_rsp_p string,
+ftp_cwd string,
+ftp_cmdarg_ts float,
+ftp_cmdarg_cmd string,
+ftp_cmdarg_arg string,
+ftp_cmdarg_seq int,
+ftp_pending_commands string,
+ftp_is_passive boolean,
+ftp_fuid string,
+ftp_last_auth_requested string,
+-- SNMP
+snmp_version string,
+snmp_community string,
+snmp_get_requests int,
+snmp_get_bulk_requests int,
+snmp_get_responses int,
+snmp_set_requests int,
+snmp_display_string string,
+snmp_up_since float,
+-- TLS
+tls_version string,
+tls_cipher string,
+tls_curve string,
+tls_server_name string,
+tls_resumed boolean,
+tls_next_protocol string,
+tls_established boolean,
+tls_cert_chain_fuids string,
+tls_client_cert_chain_fuids string,
+tls_subject string,
+tls_issuer string,
+-- SSH
+ssh_version string,
+ssh_auth_success boolean,
+ssh_client string,
+ssh_server string,
+ssh_cipher_algorithm string,
+ssh_mac_algorithm string,
+ssh_compression_algorithm string,
+ssh_key_exchange_algorithm string,
+ssh_host_key_algorithm string,
+-- DHCP
+dhcp_assigned_ip4 bigint,
+dhcp_assigned_ip4_str string,
+dhcp_mac string,
+dhcp_lease_time double,
+-- IRC
+irc_user string,
+irc_nickname string,
+irc_command string,
+irc_value string,
+irc_additional_data string,
+-- Flow
+flow_in_packets int,
+flow_out_packets int,
+flow_conn_state string,
+flow_history string,
+flow_src_dscp string,
+flow_dst_dscp string,
+flow_input string,
+flow_output string,
+-- Vulnerability
+vuln_id string,
+vuln_type string,
+vuln_status string,
+vuln_severity string,
+-- Antivirus
+av_riskname string,
+av_actualaction string,
+av_requestedaction string,
+av_secondaryaction string,
+av_downloadsite string,
+av_downloadedby string,
+av_tracking_status string,
+av_firstseen bigint,
+application_hash string,
+application_hash_type string,
+application_name string,
+application_version string,
+application_type string,
+av_categoryset string,
+av_categorytype string,
+av_threat_count int,
+av_infected_count int,
+av_omitted_count int,
+av_scanid int,
+av_startmessage string,
+av_stopmessage string,
+av_totalfiles int,
+av_signatureid string,
+av_signaturestring string,
+av_signaturesubid string,
+av_intrusionurl string,
+av_intrusionpayloadurl string,
+objectname string)
+PARTITIONED BY (
+`p_dvc_vendor` string, -- i.e. Windows, PAN, Fireeye
+`p_dvc_type` string, -- i.e. Unix, Sonicwall, Windows
+`p_dt` string -- i.e. 2017-01-01
+)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_event_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_event_pqt.sql b/spot-setup/odm/create_event_pqt.sql
new file mode 100644
index 0000000..ffd888c
--- /dev/null
+++ b/spot-setup/odm/create_event_pqt.sql
@@ -0,0 +1,301 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+-- Common
+event_time bigint,
+begintime bigint,
+endtime bigint,
+event_insertime bigint,
+lastupdatetime bigint,
+duration float,
+event_id string,
+name string,
+org string,
+type string,
+n_proto string,
+a_proto string,
+msg string,
+mac string,
+severity string,
+raw string,
+risk float,
+code string,
+category string,
+query string,
+service string,
+state string,
+in_bytes int,
+out_bytes int,
+xref string,
+version string,
+api string,
+parameter string,
+action string,
+proc string,
+app string,
+disposition string,
+prevalence string,
+confidence string,
+sensitivity string,
+count int,
+company string,
+additional_attrs map<string,string>,
+totrust string,
+fromtrust string,
+rule string,
+threat string,
+pcap_id int,
+-- Device
+dvc_time bigint,
+dvc_ip4 bigint,
+dvc_ip4_str string,
+dvc_ip6 bigint,
+dvc_ip6_str string,
+dvc_host string,
+dvc_domain string,
+dvc_type string,
+dvc_vendor string,
+dvc_fwd_ip4 bigint,
+dvc_fwd_ip4_str string,
+dvc_fwd_ip6 bigint,
+dvc_fwd_ip6_str string,
+dvc_version string,
+-- Network
+src_ip4 bigint,
+src_ip4_str string,
+src_ip6 bigint,
+src_ip6_str string,
+src_host string,
+src_domain string,
+src_port int,
+src_country_code string,
+src_country_name string,
+src_region string,
+src_city string,
+src_lat int,
+src_long int,
+dst_ip4 bigint,
+dst_ip4_str string,
+dst_ip6 bigint,
+dst_ip6_str string,
+dst_host string,
+dst_domain string,
+dst_port int,
+dst_country_code string,
+dst_country_name string,
+dst_region string,
+dst_city string,
+dst_lat int,
+dst_long int,
+src_asn int,
+dst_asn int,
+net_direction string,
+net_flags string,
+-- File
+file_name string,
+file_path string,
+file_atime bigint,
+file_acls string,
+file_type string,
+file_size int,
+file_desc string,
+file_hash string,
+file_hash_type string,
+-- Endpoint
+end_object string,
+end_action string,
+end_msg string,
+end_app string,
+end_location string,
+end_proc string,
+-- User
+user_name string,
+src_user_name string,
+dst_user_name string,
+user_email string,
+user_id string,
+user_loc string,
+user_desc string,
+-- DNS
+dns_class string,
+dns_len int,
+dns_query string,
+dns_response_code string,
+dns_answers string,
+dns_type int,
+-- Proxy
+prx_category string,
+prx_browser string,
+prx_code string,
+prx_referrer string,
+prx_host string,
+prx_filter_rule string,
+prx_filter_result string,
+prx_query string,
+prx_action string,
+prx_method string,
+prx_type string,
+-- HTTP
+http_request_method string,
+http_request_uri string,
+http_request_body_len int,
+http_request_user_name string,
+http_request_password string,
+http_request_proxied string,
+http_request_headers map<string,string>,
+http_response_status_code int,
+http_response_status_msg string,
+http_response_body_len int,
+http_response_info_code int,
+http_response_info_msg string,
+http_response_resp_fuids string,
+http_response_mime_types string,
+http_response_headers map<string,string>,
+-- SMTP
+smtp_trans_depth int,
+smtp_headers_helo string,
+smtp_headers_mailfrom string,
+smtp_headers_rcptto string,
+smtp_headers_date string,
+smtp_headers_from string,
+smtp_headers_to string,
+smtp_headers_reply_to string,
+smtp_headers_msg_id string,
+smtp_headers_in_reply_to string,
+smtp_headers_subject string,
+smtp_headers_x_originating_ip4 bigint,
+smtp_headers_x_originating_ip4_str string,
+smtp_headers_first_received string,
+smtp_headers_second_received string,
+smtp_last_reply string,
+smtp_path string,
+smtp_user_agent string,
+smtp_tls boolean,
+smtp_is_webmail boolean,
+-- FTP
+ftp_user_name string,
+ftp_password string,
+ftp_command string,
+ftp_arg string,
+ftp_mime_type string,
+ftp_file_size int,
+ftp_reply_code int,
+ftp_reply_msg string,
+ftp_data_channel_passive boolean,
+ftp_data_channel_rsp_p string,
+ftp_cwd string,
+ftp_cmdarg_ts float,
+ftp_cmdarg_cmd string,
+ftp_cmdarg_arg string,
+ftp_cmdarg_seq int,
+ftp_pending_commands string,
+ftp_is_passive boolean,
+ftp_fuid string,
+ftp_last_auth_requested string,
+-- SNMP
+snmp_version string,
+snmp_community string,
+snmp_get_requests int,
+snmp_get_bulk_requests int,
+snmp_get_responses int,
+snmp_set_requests int,
+snmp_display_string string,
+snmp_up_since float,
+-- TLS
+tls_version string,
+tls_cipher string,
+tls_curve string,
+tls_server_name string,
+tls_resumed boolean,
+tls_next_protocol string,
+tls_established boolean,
+tls_cert_chain_fuids string,
+tls_client_cert_chain_fuids string,
+tls_subject string,
+tls_issuer string,
+-- SSH
+ssh_version string,
+ssh_auth_success boolean,
+ssh_client string,
+ssh_server string,
+ssh_cipher_algorithm string,
+ssh_mac_algorithm string,
+ssh_compression_algorithm string,
+ssh_key_exchange_algorithm string,
+ssh_host_key_algorithm string,
+-- DHCP
+dhcp_assigned_ip4 bigint,
+dhcp_assigned_ip4_str string,
+dhcp_mac string,
+dhcp_lease_time double,
+-- IRC
+irc_user string,
+irc_nickname string,
+irc_command string,
+irc_value string,
+irc_additional_data string,
+-- Flow
+flow_in_packets int,
+flow_out_packets int,
+flow_conn_state string,
+flow_history string,
+flow_src_dscp string,
+flow_dst_dscp string,
+flow_input string,
+flow_output string,
+-- Vulnerability
+vuln_id string,
+vuln_type string,
+vuln_status string,
+vuln_severity string,
+-- Antivirus
+av_riskname string,
+av_actualaction string,
+av_requestedaction string,
+av_secondaryaction string,
+av_downloadsite string,
+av_downloadedby string,
+av_tracking_status string,
+av_firstseen bigint,
+application_hash string,
+application_hash_type string,
+application_name string,
+application_version string,
+application_type string,
+av_categoryset string,
+av_categorytype string,
+av_threat_count int,
+av_infected_count int,
+av_omitted_count int,
+av_scanid int,
+av_startmessage string,
+av_stopmessage string,
+av_totalfiles int,
+av_signatureid string,
+av_signaturestring string,
+av_signaturesubid string,
+av_intrusionurl string,
+av_intrusionpayloadurl string,
+objectname string)
+PARTITIONED BY (
+`p_dvc_vendor` string, -- i.e. Windows, PAN, Fireeye
+`p_dvc_type` string, -- i.e. Unix, Sonicwall, Windows
+`p_dt` string -- i.e. 2017-01-01
+)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_network_context_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_network_context_avro.sql b/spot-setup/odm/create_network_context_avro.sql
new file mode 100644
index 0000000..c4d1f4c
--- /dev/null
+++ b/spot-setup/odm/create_network_context_avro.sql
@@ -0,0 +1,48 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+net_domain_name string,
+net_registry_domain_id string,
+net_registrar_whois_server string,
+net_registrar_url string,
+net_update_date bigint,
+net_creation_date bigint,
+net_registrar_registration_expiration_date bigint,
+net_registrar string,
+net_registrar_iana_id string,
+net_registrar_abuse_contact_email string,
+net_registrar_abuse_contact_phone string,
+net_domain_status string,
+net_registry_registrant_id string,
+net_registrant_name string,
+net_registrant_organization string,
+net_registrant_street string,
+net_registrant_city string,
+net_registrant_state string,
+net_registrant_post_code string,
+net_registrant_country string,
+net_registrant_phone string,
+net_registrant_email string,
+net_registry_admin_id string,
+net_name_servers string,
+net_dnssec string,
+net_risk float)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}')
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_network_context_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_network_context_pqt.sql b/spot-setup/odm/create_network_context_pqt.sql
new file mode 100644
index 0000000..0a87e7c
--- /dev/null
+++ b/spot-setup/odm/create_network_context_pqt.sql
@@ -0,0 +1,47 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+net_domain_name string,
+net_registry_domain_id string,
+net_registrar_whois_server string,
+net_registrar_url string,
+net_update_date bigint,
+net_creation_date bigint,
+net_registrar_registration_expiration_date bigint,
+net_registrar string,
+net_registrar_iana_id string,
+net_registrar_abuse_contact_email string,
+net_registrar_abuse_contact_phone string,
+net_domain_status string,
+net_registry_registrant_id string,
+net_registrant_name string,
+net_registrant_organization string,
+net_registrant_street string,
+net_registrant_city string,
+net_registrant_state string,
+net_registrant_post_code string,
+net_registrant_country string,
+net_registrant_phone string,
+net_registrant_email string,
+net_registry_admin_id string,
+net_name_servers string,
+net_dnssec string,
+net_risk float)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}'
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_threat_intelligence_context_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_threat_intelligence_context_avro.sql b/spot-setup/odm/create_threat_intelligence_context_avro.sql
new file mode 100644
index 0000000..dbb648b
--- /dev/null
+++ b/spot-setup/odm/create_threat_intelligence_context_avro.sql
@@ -0,0 +1,76 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+ti_source string,
+ti_provider_id string,
+ti_indicator_id string,
+ti_indicator_desc string,
+ti_date_added bigint,
+ti_date_modified bigint,
+ti_risk_impact string,
+ti_severity string,
+ti_category string,
+ti_campaign_name string,
+ti_deployed_location array<string>,
+ti_associated_incidents string,
+ti_adversarial_identification_group string,
+ti_adversarial_identification_tactics string,
+ti_adversarial_identification_reports string,
+ti_phase string,
+ti_indicator_cve string,
+ti_indicator_ip4 array<bigint>,
+ti_indicator_ip4_str array<string>,
+ti_indicator_ip6 array<bigint>,
+ti_indicator_ip6_str array<string>,
+ti_indicator_domain string,
+ti_indicator_hostname string,
+ti_indicator_email array<string>,
+ti_indicator_url array<string>,
+ti_indicator_uri array<string>,
+ti_indicator_file_hash string,
+ti_indicator_file_path string,
+ti_indicator_mutex string,
+ti_indicator_md5 string,
+ti_indicator_sha1 string,
+ti_indicator_sha256 string,
+ti_indicator_device_path string,
+ti_indicator_drive string,
+ti_indicator_file_name string,
+ti_indicator_file_extension string,
+ti_indicator_file_size string,
+ti_indicator_file_created bigint,
+ti_indicator_file_accessed bigint,
+ti_indicator_file_changed bigint,
+ti_indicator_file_entropy string,
+ti_indicator_file_attributes array<string>,
+ti_indicator_user_name string,
+ti_indicator_security_id string,
+ti_indicator_pe_info array<string>,
+ti_indicator_pe_type array<string>,
+ti_indicator_strings array<string>,
+ti_indicator_org string,
+ti_indicator_reg_name string,
+ti_indicator_reg_email string,
+ti_indicator_reg_org string,
+ti_indicator_reg_phone string,
+ti_tags string,
+ti_threat_type string)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}')
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_threat_intelligence_context_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_threat_intelligence_context_pqt.sql b/spot-setup/odm/create_threat_intelligence_context_pqt.sql
new file mode 100644
index 0000000..156169e
--- /dev/null
+++ b/spot-setup/odm/create_threat_intelligence_context_pqt.sql
@@ -0,0 +1,75 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+ti_source string,
+ti_provider_id string,
+ti_indicator_id string,
+ti_indicator_desc string,
+ti_date_added bigint,
+ti_date_modified bigint,
+ti_risk_impact string,
+ti_severity string,
+ti_category string,
+ti_campaign_name string,
+ti_deployed_location array<string>,
+ti_associated_incidents string,
+ti_adversarial_identification_group string,
+ti_adversarial_identification_tactics string,
+ti_adversarial_identification_reports string,
+ti_phase string,
+ti_indicator_cve string,
+ti_indicator_ip4 array<bigint>,
+ti_indicator_ip4_str array<string>,
+ti_indicator_ip6 array<bigint>,
+ti_indicator_ip6_str array<string>,
+ti_indicator_domain string,
+ti_indicator_hostname string,
+ti_indicator_email array<string>,
+ti_indicator_url array<string>,
+ti_indicator_uri array<string>,
+ti_indicator_file_hash string,
+ti_indicator_file_path string,
+ti_indicator_mutex string,
+ti_indicator_md5 string,
+ti_indicator_sha1 string,
+ti_indicator_sha256 string,
+ti_indicator_device_path string,
+ti_indicator_drive string,
+ti_indicator_file_name string,
+ti_indicator_file_extension string,
+ti_indicator_file_size string,
+ti_indicator_file_created bigint,
+ti_indicator_file_accessed bigint,
+ti_indicator_file_changed bigint,
+ti_indicator_file_entropy string,
+ti_indicator_file_attributes array<string>,
+ti_indicator_user_name string,
+ti_indicator_security_id string,
+ti_indicator_pe_info array<string>,
+ti_indicator_pe_type array<string>,
+ti_indicator_strings array<string>,
+ti_indicator_org string,
+ti_indicator_reg_name string,
+ti_indicator_reg_email string,
+ti_indicator_reg_org string,
+ti_indicator_reg_phone string,
+ti_tags string,
+ti_threat_type string)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}'
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_user_context_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_user_context_avro.sql b/spot-setup/odm/create_user_context_avro.sql
new file mode 100644
index 0000000..1e73f19
--- /dev/null
+++ b/spot-setup/odm/create_user_context_avro.sql
@@ -0,0 +1,51 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+dvc_time bigint,
+user_created bigint,
+user_changed bigint,
+user_last_logon bigint,
+user_logon_count int,
+user_last_reset bigint,
+user_expiration bigint,
+user_image binary,
+user_id string,
+user_name string,
+user_name_first string,
+user_name_middle string,
+user_name_last string,
+user_name_mgr string,
+user_phone string,
+user_email string,
+user_code string,
+user_loc string,
+user_departm string,
+user_dn string,
+user_ou string,
+user_empid string,
+user_title string,
+user_groups array<string>,
+dvc_type string,
+dvc_vendor string,
+user_risk float,
+dvc_version string,
+additional_attrs map<string,string>)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}')
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_user_context_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_user_context_pqt.sql b/spot-setup/odm/create_user_context_pqt.sql
new file mode 100644
index 0000000..21843ed
--- /dev/null
+++ b/spot-setup/odm/create_user_context_pqt.sql
@@ -0,0 +1,50 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+dvc_time bigint,
+user_created bigint,
+user_changed bigint,
+user_last_logon bigint,
+user_logon_count int,
+user_last_reset bigint,
+user_expiration bigint,
+user_img binary,
+user_id string,
+user_name string,
+user_name_first string,
+user_name_middle string,
+user_name_last string,
+user_name_mgr string,
+user_phone string,
+user_email string,
+user_code string,
+user_loc string,
+user_departm string,
+user_dn string,
+user_ou string,
+user_empid string,
+user_title string,
+user_groups array<string>,
+dvc_type string,
+dvc_vendor string,
+user_risk float,
+dvc_version string,
+additional_attrs map<string,string>)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}'
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_vulnerability_context_avro.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_vulnerability_context_avro.sql b/spot-setup/odm/create_vulnerability_context_avro.sql
new file mode 100644
index 0000000..f11d89f
--- /dev/null
+++ b/spot-setup/odm/create_vulnerability_context_avro.sql
@@ -0,0 +1,32 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+vuln_id string,
+vuln_title string,
+vuln_description string,
+vuln_solution string,
+vuln_type string,
+vuln_category string,
+vuln_severity string,
+vuln_created bigint,
+vuln_updated bigint,
+additional_attrs map<string,string>)
+STORED AS AVRO
+LOCATION '${VAR:ODM_LOCATION}'
+TBLPROPERTIES ('avro.schema.url'='${VAR:ODM_AVRO_URL}')
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/create_vulnerability_context_pqt.sql
----------------------------------------------------------------------
diff --git a/spot-setup/odm/create_vulnerability_context_pqt.sql b/spot-setup/odm/create_vulnerability_context_pqt.sql
new file mode 100644
index 0000000..2629c10
--- /dev/null
+++ b/spot-setup/odm/create_vulnerability_context_pqt.sql
@@ -0,0 +1,31 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME};
+CREATE EXTERNAL TABLE IF NOT EXISTS ${VAR:ODM_DBNAME}.${VAR:ODM_TABLENAME} (
+vuln_id string,
+vuln_title string,
+vuln_description string,
+vuln_solution string,
+vuln_type string,
+vuln_category string,
+vuln_severity string,
+vuln_created bigint,
+vuln_updated bigint,
+additional_attrs map<string,string>)
+STORED AS PARQUET
+LOCATION '${VAR:ODM_LOCATION}'
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/endpoint_context.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/endpoint_context.avsc b/spot-setup/odm/endpoint_context.avsc
new file mode 100644
index 0000000..9fa923c
--- /dev/null
+++ b/spot-setup/odm/endpoint_context.avsc
@@ -0,0 +1,44 @@
+{
+ "namespace":"org.apache.spot",
+ "name":"endpoint_context",
+ "type": "record",
+ "fields": [
+ {"name":"dvc_time","type":["null","long"],"doc":"Timestamp from when the endpoint context information is obtained","default":null},
+ {"name":"end_ip4", "type":["null","long"],"doc":"IP address of endpoint","default":null},
+ {"name":"end_ip4_str", "type":["null","string"],"doc":"IP address of endpoint","default":null},
+ {"name":"end_ip6", "type":["null","long"],"doc":"IP address of endpoint","default":null},
+ {"name":"end_ip6_str", "type":["null","string"],"doc":"IP address of endpoint","default":null},
+ {"name":"end_os", "type":["null","string"],"doc":"Operating system","default":null},
+ {"name":"end_os_version", "type":["null","string"],"doc":"Version of OS","default":null},
+ {"name":"end_os_sp", "type":["null","string"],"doc":"Service pack","default":null},
+ {"name":"end_tz", "type":["null","string"],"doc":"Timezone","default":null},
+ {"name":"end_hotfixes", "type":["null",{"type":"array", "items":"string"}],"doc":"Applied hotfixes","default":null},
+ {"name":"end_disks", "type":["null",{"type":"array", "items":"string"}],"doc":"Available disks","default":null},
+ {"name":"end_removeables", "type":["null",{"type":"array", "items":"string"}],"doc":"Removable media devices","default":null},
+ {"name":"end_nics", "type":["null",{"type":"array", "items":"string"}],"doc":"Network interfaces","default":null},
+ {"name":"end_drivers", "type":["null",{"type":"array", "items":"string"}],"doc":"Installed kernel drivers","default":null},
+ {"name":"end_users", "type":["null",{"type":"array", "items":"string"}],"doc":"Local user accounts","default":null},
+ {"name":"end_host", "type":["null","string"],"doc":"Hostname of endpoint","default":null},
+ {"name":"end_mac", "type":["null","string"],"doc":"MAC address of endpoint","default":null},
+ {"name":"end_owner", "type":["null","string"],"doc":"Endpoint owner (name)","default":null},
+ {"name":"end_vulns", "type":["null",{"type":"array", "items":"string"}],"doc":"Vulnerability identifiers (CVE identifier)","default":null},
+ {"name":"end_loc", "type":["null","string"],"doc":"Location","default":null},
+ {"name":"end_departm", "type":["null","string"],"doc":"Department","default":null},
+ {"name":"end_company", "type":["null","string"],"doc":"Distinguished name","default":null},
+ {"name":"end_regs", "type":["null",{"type":"array", "items":"string"}],"doc":"Applicable regulations","default":null},
+ {"name":"end_svcs", "type":["null",{"type":"array", "items":"string"}],"doc":"Services running on system","default":null},
+ {"name":"end_procs", "type":["null",{"type":"array", "items":"string"}],"doc":"Processes","default":null},
+ {"name":"end_criticality", "type":["null","string"],"doc":"Criticality of device","default":null},
+ {"name":"end_apps", "type":["null",{"type":"array", "items":"string"}],"doc":"Applications running on system","default":null},
+ {"name":"end_desc", "type":["null","string"],"doc":"Endpoint descriptor","default":null},
+ {"name":"dvc_type", "type":["null","string"],"doc":"Device type that generated the log","default":null},
+ {"name":"dvc_vendor", "type":["null","string"],"doc":"Vendor","default":null},
+ {"name":"dvc_version", "type":["null","string"],"doc":"Version","default":null},
+ {"name":"end_architecture", "type":["null","string"],"doc":"CPU architecture","default":null},
+ {"name":"end_uuid", "type":["null","string"],"doc":"Universally unique identifier","default":null},
+ {"name":"end_risk", "type":["null","float"],"doc":"Risk score","default":null},
+ {"name":"end_memtotal", "type":["null","int"],"doc":"Total memory (bytes)","default":null},
+ {"name":"additional_attrs", "type":["null",{"type":"map","values":["null","string"]}],"default":null,"doc":"Additional attributes of endpoint"}
+ ],
+ "doc": "A view schema for storing Apache Spot Endpoint Context data."
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/event.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/event.avsc b/spot-setup/odm/event.avsc
new file mode 100644
index 0000000..50dc033
--- /dev/null
+++ b/spot-setup/odm/event.avsc
@@ -0,0 +1,266 @@
+{
+
+ "namespace":"org.apache.spot",
+ "name":"event",
+ "type": "record",
+ "fields": [
+ {"name":"event_time","type":["null","long"],"doc":"timestamp of event (UTC)", "default": null},
+ {"name":"begintime","type":["null","long"],"doc":"timestamp of event (UTC)", "default": null},
+ {"name":"endtime","type":["null","long"],"doc":"timestamp of event (UTC)", "default": null},
+ {"name":"event_insertime","type":["null","long"],"doc":"timestamp of event (UTC)", "default": null},
+ {"name":"lastupdatetime","type":["null","long"],"doc":"timestamp of event (UTC)", "default": null},
+ {"name":"duration", "type":["null","float"],"doc":"Time duration (milliseconds)", "default": null},
+ {"name":"event_id", "type":["null","string"],"doc":"Unique identifier for event", "default": null},
+ {"name":"name", "type":["null","string"],"doc":"Name of event", "default": null},
+ {"name":"org", "type":["null","string"],"doc":"Organization", "default": null},
+ {"name":"type", "type":["null","string"],"doc":"Type information", "default": null},
+ {"name":"n_proto", "type":["null","string"],"doc":"Network protocol of event", "default": null},
+ {"name":"a_proto", "type":["null","string"],"doc":"Application protocol of event", "default": null},
+ {"name":"msg", "type":["null","string"],"doc":"Message (details of action taken on object)", "default": null},
+ {"name":"mac", "type":["null","string"],"doc":"MAC address", "default": null},
+ {"name":"severity", "type":["null","string"],"doc":"Severity of event", "default": null},
+ {"name":"raw", "type":["null","string"],"doc":"Raw text message of entire event", "default": null},
+ {"name":"risk", "type":["null","float"],"doc":"Risk score", "default": null},
+ {"name":"code", "type":["null","string"],"doc":"Response or error code", "default": null},
+ {"name":"category", "type":["null","string"],"doc":"Event category", "default": null},
+ {"name":"query", "type":["null","string"],"doc":"Query (DNS query, URI query, SQL query, etc.)", "default": null},
+ {"name":"service", "type":["null","string"],"doc":"(i.e. service name, type of service)", "default": null},
+ {"name":"state", "type":["null","string"],"doc":"State of object", "default": null},
+ {"name":"in_bytes", "type":["null","int"],"doc":"Bytes in", "default": null},
+ {"name":"out_bytes", "type":["null","int"],"doc":"Bytes out", "default": null},
+ {"name":"xref", "type":["null","string"],"doc":"External reference to public description", "default": null},
+ {"name":"version", "type":["null","string"],"doc":"Version", "default": null},
+ {"name":"api", "type":["null","string"],"doc":"API label", "default": null},
+ {"name":"parameter", "type":["null","string"],"doc":"Parameter label", "default": null},
+ {"name":"action", "type":["null","string"],"doc":"Action label", "default": null},
+ {"name":"proc", "type":["null","string"],"doc":"Process label", "default": null},
+ {"name":"app", "type":["null","string"],"doc":"Application label", "default": null},
+ {"name":"disposition", "type":["null","string"],"doc":"Disposition label", "default": null},
+ {"name":"prevalence", "type":["null","string"],"doc":"Prevalence label", "default": null},
+ {"name":"confidence", "type":["null","string"],"doc":"Confidence label", "default": null},
+ {"name":"sensitivity", "type":["null","string"],"doc":"Sensitivity label", "default": null},
+ {"name":"count", "type":["null","int"],"doc":"Generic count", "default": null},
+ {"name":"company", "type":["null","string"],"doc":"Company label", "default": null},
+ {"name":"additional_attrs","type":["null",{"type":"map","values":["null","string"]}],"default":null, "doc":"Additional attributes of the event"},
+ {"name":"totrust", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"fromtrust", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"rule", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"threat", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"pcap_id", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"dvc_time", "type":["null","long"],"doc":"UTC timestamp from device where event/alert originates or is received", "default": null},
+ {"name":"dvc_ip4", "type":["null","long"],"doc":"IP address of device", "default": null},
+ {"name":"dvc_ip4_str", "type":["null","string"],"doc":"IP address of device", "default": null},
+ {"name":"dvc_ip6", "type":["null","long"],"doc":"IP address of device", "default": null},
+ {"name":"dvc_ip6_str", "type":["null","string"],"doc":"IP address of device", "default": null},
+ {"name":"dvc_host", "type":["null","string"],"doc":"Hostname of device", "default": null},
+ {"name":"dvc_domain", "type":["null","string"],"doc":"Domain of device", "default": null},
+ {"name":"dvc_type", "type":["null","string"],"doc":"Device type that generated the log", "default": null},
+ {"name":"dvc_vendor", "type":["null","string"],"doc":"Vendor", "default": null},
+ {"name":"dvc_fwd_ip4", "type":["null","long"],"doc":"Forwarded from device", "default": null},
+ {"name":"dvc_fwd_ip4_str", "type":["null","string"],"doc":"Forwarded from device", "default": null},
+ {"name":"dvc_fwd_ip6", "type":["null","long"],"doc":"Forwarded from device", "default": null},
+ {"name":"dvc_fwd_ip6_str", "type":["null","string"],"doc":"Forwarded from device", "default": null},
+ {"name":"dvc_version", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"src_ip4", "type":["null","long"],"doc":"Source ip address of event", "default": null},
+ {"name":"src_ip4_str", "type":["null","string"],"doc":"Source ip address of event", "default": null},
+ {"name":"src_ip6", "type":["null","long"],"doc":"Source ip address of event", "default": null},
+ {"name":"src_ip6_str", "type":["null","string"],"doc":"Source ip address of event", "default": null},
+ {"name":"src_host", "type":["null","string"],"doc":"Source FQDN of event", "default": null},
+ {"name":"src_domain", "type":["null","string"],"doc":"Domain name of source address", "default": null},
+ {"name":"src_port", "type":["null","int"],"doc":"Source port of event", "default": null},
+ {"name":"src_country_code", "type":["null","string"],"doc":"Source country code", "default": null},
+ {"name":"src_country_name", "type":["null","string"],"doc":"Source country name", "default": null},
+ {"name":"src_region", "type":["null","string"],"doc":"Source region", "default": null},
+ {"name":"src_city", "type":["null","string"],"doc":"Source city", "default": null},
+ {"name":"src_lat", "type":["null","int"],"doc":"Source latitude", "default": null},
+ {"name":"src_long", "type":["null","int"],"doc":"Source longitude", "default": null},
+ {"name":"dst_ip4", "type":["null","long"],"doc":"Destination ip address of event", "default": null},
+ {"name":"dst_ip4_str", "type":["null","string"],"doc":"Destination ip address of event", "default": null},
+ {"name":"dst_ip6", "type":["null","long"],"doc":"Destination ip address of event", "default": null},
+ {"name":"dst_ip6_str", "type":["null","string"],"doc":"Destination ip address of event", "default": null},
+ {"name":"dst_host", "type":["null","string"],"doc":"Destination FQDN of event", "default": null},
+ {"name":"dst_domain", "type":["null","string"],"doc":"Domain name of destination address", "default": null},
+ {"name":"dst_port", "type":["null","int"],"doc":"Destination port of event", "default": null},
+ {"name":"dst_country_code", "type":["null","string"],"doc":"Source country code", "default": null},
+ {"name":"dst_country_name", "type":["null","string"],"doc":"Source country name", "default": null},
+ {"name":"dst_region", "type":["null","string"],"doc":"Source region", "default": null},
+ {"name":"dst_city", "type":["null","string"],"doc":"Source city", "default": null},
+ {"name":"dst_lat", "type":["null","int"],"doc":"Source latitude", "default": null},
+ {"name":"dst_long", "type":["null","int"],"doc":"Source longitude", "default": null},
+ {"name":"src_asn", "type":["null","int"],"doc":"Autonomous system number", "default": null},
+ {"name":"dst_asn", "type":["null","int"],"doc":"Autonomous system number", "default": null},
+ {"name":"net_direction", "type":["null","string"],"doc":"Direction", "default": null},
+ {"name":"net_flags", "type":["null","string"],"doc":"TCP flags", "default": null},
+ {"name":"file_name", "type":["null","string"],"doc":"Filename from event", "default": null},
+ {"name":"file_path", "type":["null","string"],"doc":"File path", "default": null},
+ {"name":"file_atime", "type":["null","long"],"doc":"Timestamp (UTC) of file access", "default": null},
+ {"name":"file_acls", "type":["null","string"],"doc":"File permissions", "default": null},
+ {"name":"file_type", "type":["null","string"],"doc":"Type of file", "default": null},
+ {"name":"file_size", "type":["null","int"],"doc":"Size of file in bytes", "default": null},
+ {"name":"file_desc", "type":["null","string"],"doc":"Description of file", "default": null},
+ {"name":"file_hash", "type":["null","string"],"doc":"Hash of file", "default": null},
+ {"name":"file_hash_type", "type":["null","string"],"doc":"Type of hash", "default": null},
+ {"name":"end_object", "type":["null","string"],"doc":"File/Process/ Registry", "default": null},
+ {"name":"end_action", "type":["null","string"],"doc":"Action taken on object (open/delete/ edit)", "default": null},
+ {"name":"end_msg", "type":["null","string"],"doc":"Message (details of action taken on object)", "default": null},
+ {"name":"end_app", "type":["null","string"],"doc":"Application", "default": null},
+ {"name":"end_location", "type":["null","string"],"doc":"Location", "default": null},
+ {"name":"end_proc", "type":["null","string"],"doc":"Process", "default": null},
+ {"name":"user_name", "type":["null","string"],"doc":"username from event", "default": null},
+ {"name":"src_user_name", "type":["null","string"],"doc":"username from event", "default": null},
+ {"name":"dst_user_name", "type":["null","string"],"doc":"username from event", "default": null},
+ {"name":"user_email", "type":["null","string"],"doc":"Email address", "default": null},
+ {"name":"user_id", "type":["null","string"],"doc":"userid", "default": null},
+ {"name":"user_loc", "type":["null","string"],"doc":"location", "default": null},
+ {"name":"user_desc", "type":["null","string"],"doc":"Description of user", "default": null},
+ {"name":"dns_class", "type":["null","string"],"doc":"DNS class", "default": null},
+ {"name":"dns_len", "type":["null","int"],"doc":"DNS frame length", "default": null},
+ {"name":"dns_query", "type":["null","string"],"doc":"Requested DNS query", "default": null},
+ {"name":"dns_response_code", "type":["null","string"],"doc":"Response code", "default": null},
+ {"name":"dns_answers", "type":["null","string"],"doc":"Response to DNS Query", "default": null},
+ {"name":"dns_type", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"prx_category", "type":["null","string"],"doc":"Event category", "default": null},
+ {"name":"prx_browser", "type":["null","string"],"doc":"Web browser", "default": null},
+ {"name":"prx_code", "type":["null","string"],"doc":"Error or response code", "default": null},
+ {"name":"prx_referrer", "type":["null","string"],"doc":"Referrer", "default": null},
+ {"name":"prx_host", "type":["null","string"],"doc":"Requested URI", "default": null},
+ {"name":"prx_filter_rule", "type":["null","string"],"doc":"Applied filter or rule", "default": null},
+ {"name":"prx_filter_result", "type":["null","string"],"doc":"Result of applied filter or rule", "default": null},
+ {"name":"prx_query", "type":["null","string"],"doc":"URI query", "default": null},
+ {"name":"prx_action", "type":["null","string"],"doc":"Action taken on object", "default": null},
+ {"name":"prx_method", "type":["null","string"],"doc":"HTTP method", "default": null},
+ {"name":"prx_type", "type":["null","string"],"doc":"Type of request", "default": null},
+ {"name":"http_request_method", "type":["null","string"],"doc":"HTTP method", "default": null},
+ {"name":"http_request_uri", "type":["null","string"],"doc":"Requested URI", "default": null},
+ {"name":"http_request_body_len", "type":["null","int"],"doc":"Length of request body", "default": null},
+ {"name":"http_request_user_name", "type":["null","string"],"doc":"username from event", "default": null},
+ {"name":"http_request_password", "type":["null","string"],"doc":"Password from event", "default": null},
+ {"name":"http_request_proxied", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"http_request_headers", "type":["null",{"type":"map","values":["null","string"]}],"default":null,"doc":"HTTP request headers"},
+ {"name":"http_response_status_code", "type":["null","int"],"doc":"HTTP response status code", "default": null},
+ {"name":"http_response_status_msg", "type":["null","string"],"doc":"HTTP response status message", "default": null},
+ {"name":"http_response_body_len", "type":["null","int"],"doc":"Length of response body", "default": null},
+ {"name":"http_response_info_code", "type":["null","int"],"doc":"HTTP response info code", "default": null},
+ {"name":"http_response_info_msg", "type":["null","string"],"doc":"HTTP response info message", "default": null},
+ {"name":"http_response_resp_fuids", "type":["null","string"],"doc":"Response FUIDS", "default": null},
+ {"name":"http_response_mime_types", "type":["null","string"],"doc":"Mime types", "default": null},
+ {"name":"http_response_headers", "type":["null",{"type":"map","values":["null","string"]}],"default":null,"doc":"Response headers"},
+ {"name":"smtp_trans_depth", "type":["null","int"],"doc":"Depth of email into SMTP exchange", "default": null},
+ {"name":"smtp_headers_helo", "type":["null","string"],"doc":"Helo header", "default": null},
+ {"name":"smtp_headers_mailfrom", "type":["null","string"],"doc":"Mailfrom header", "default": null},
+ {"name":"smtp_headers_rcptto", "type":["null","string"],"doc":"Rcptto header", "default": null},
+ {"name":"smtp_headers_date", "type":["null","string"],"doc":"Header date", "default": null},
+ {"name":"smtp_headers_from", "type":["null","string"],"doc":"From header", "default": null},
+ {"name":"smtp_headers_to", "type":["null","string"],"doc":"To header", "default": null},
+ {"name":"smtp_headers_reply_to", "type":["null","string"],"doc":"Reply to header", "default": null},
+ {"name":"smtp_headers_msg_id", "type":["null","string"],"doc":"Message ID", "default": null},
+ {"name":"smtp_headers_in_reply_to", "type":["null","string"],"doc":"In reply to header", "default": null},
+ {"name":"smtp_headers_subject", "type":["null","string"],"doc":"Subject", "default": null},
+ {"name":"smtp_headers_x_originating_ip4", "type":["null","long"],"doc":"Originating IP address", "default": null},
+ {"name":"smtp_headers_x_originating_ip4_str", "type":["null","string"],"doc":"Originating IP address", "default": null},
+ {"name":"smtp_headers_first_received", "type":["null","string"],"doc":"First to receive message", "default": null},
+ {"name":"smtp_headers_second_received", "type":["null","string"],"doc":"Second to receive message", "default": null},
+ {"name":"smtp_last_reply", "type":["null","string"],"doc":"Last reply in message chain", "default": null},
+ {"name":"smtp_path", "type":["null","string"],"doc":"Path of message", "default": null},
+ {"name":"smtp_user_agent", "type":["null","string"],"doc":"User agent", "default": null},
+ {"name":"smtp_tls", "type":["null","boolean"],"doc":"Indication of TLS use", "default": null},
+ {"name":"smtp_is_webmail", "type":["null","boolean"],"doc":"Indication of webmail", "default": null},
+ {"name":"ftp_user_name", "type":["null","string"],"doc":"Username", "default": null},
+ {"name":"ftp_password", "type":["null","string"],"doc":"Password", "default": null},
+ {"name":"ftp_command", "type":["null","string"],"doc":"FTP command", "default": null},
+ {"name":"ftp_arg", "type":["null","string"],"doc":"Argument", "default": null},
+ {"name":"ftp_mime_type", "type":["null","string"],"doc":"Mime type", "default": null},
+ {"name":"ftp_file_size", "type":["null","int"],"doc":"File size", "default": null},
+ {"name":"ftp_reply_code", "type":["null","int"],"doc":"Reply code", "default": null},
+ {"name":"ftp_reply_msg", "type":["null","string"],"doc":"Reply message", "default": null},
+ {"name":"ftp_data_channel_passive", "type":["null","boolean"],"doc":"Passive data channel?", "default": null},
+ {"name":"ftp_data_channel_rsp_p", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ftp_cwd", "type":["null","string"],"doc":"Current working directory", "default": null},
+ {"name":"ftp_cmdarg_ts", "type":["null","float"],"doc":"TBD", "default": null},
+ {"name":"ftp_cmdarg_cmd", "type":["null","string"],"doc":"Command", "default": null},
+ {"name":"ftp_cmdarg_arg", "type":["null","string"],"doc":"Command argument", "default": null},
+ {"name":"ftp_cmdarg_seq", "type":["null","int"],"doc":"Sequence", "default": null},
+ {"name":"ftp_pending_commands", "type":["null","string"],"doc":"Pending commands", "default": null},
+ {"name":"ftp_is_passive", "type":["null","boolean"],"doc":"Passive mode enabled", "default": null},
+ {"name":"ftp_fuid", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ftp_last_auth_requested", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"snmp_version", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"snmp_community", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"snmp_get_requests", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"snmp_get_bulk_requests", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"snmp_get_responses", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"snmp_set_requests", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"snmp_display_string", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"snmp_up_since", "type":["null","float"],"doc":"TBD", "default": null},
+ {"name":"tls_version", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_cipher", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_curve", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_server_name", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_resumed", "type":["null","boolean"],"doc":"TBD", "default": null},
+ {"name":"tls_next_protocol", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_established", "type":["null","boolean"],"doc":"TBD", "default": null},
+ {"name":"tls_cert_chain_fuids", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_client_cert_chain_fuids", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_subject", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"tls_issuer", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_version", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_auth_success", "type":["null","boolean"],"doc":"TBD", "default": null},
+ {"name":"ssh_client", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_server", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_cipher_algorithm", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_mac_algorithm", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_compression_algorithm", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_key_exchange_algorithm", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"ssh_host_key_algorithm", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"dhcp_assigned_ip4", "type":["null","long"],"doc":"TBD", "default": null},
+ {"name":"dhcp_assigned_ip4_str", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"dhcp_mac", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"dhcp_lease_time", "type":["null","double"],"doc":"TBD", "default": null},
+ {"name":"irc_user", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"irc_nickname", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"irc_command", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"irc_value", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"irc_additional_data", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_in_packets", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"flow_out_packets", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"flow_conn_state", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_history", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_src_dscp", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_dst_dscp", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_input", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"flow_output", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_id", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_type", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_status", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"vuln_severity", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_riskname", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_actualaction", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_requestedaction", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_secondaryaction", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_downloadsite", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_downloadedby", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_tracking_status", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_firstseen", "type":["null","long"],"doc":"TBD", "default": null},
+ {"name":"application_hash", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"application_hash_type", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"application_name", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"application_version", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"application_type", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_categoryset", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_categorytype", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_threat_count", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"av_infected_count", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"av_omitted_count", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"av_scanid", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"av_startmessage", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_stopmessage", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_totalfiles", "type":["null","int"],"doc":"TBD", "default": null},
+ {"name":"av_signatureid", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_signaturestring", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_signaturesubid", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_intrusionurl", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"av_intrusionpayloadurl", "type":["null","string"],"doc":"TBD", "default": null},
+ {"name":"objectname", "type":["null","string"],"doc":"TBD", "default": null}
+ ],
+ "doc": "A view schema for storing Apache Spot Event data."
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/network_context.avsc
----------------------------------------------------------------------
diff --git a/spot-setup/odm/network_context.avsc b/spot-setup/odm/network_context.avsc
new file mode 100644
index 0000000..85bc335
--- /dev/null
+++ b/spot-setup/odm/network_context.avsc
@@ -0,0 +1,34 @@
+{
+ "namespace":"org.apache.spot",
+ "name":"network_context",
+ "type": "record",
+ "fields": [
+ {"name":"net_domain_name","type":["null","string"],"doc":"Domain name", "default": null},
+ {"name":"net_registry_domain_id", "type":["null","string"],"doc":"Registry Domain ID", "default": null},
+ {"name":"net_registrar_whois_server", "type":["null","string"],"doc":"Registrar WHOIS Server", "default": null},
+ {"name":"net_registrar_url", "type":["null","string"],"doc":"Registrar URL", "default": null},
+ {"name":"net_update_date", "type":["null","long"],"doc":"UTC timestamp", "default": null},
+ {"name":"net_creation_date", "type":["null","long"],"doc":"Creation Date", "default": null},
+ {"name":"net_registrar_registration_expiration_date", "type":["null","long"],"doc":"Registrar Registration Expiration Date", "default": null},
+ {"name":"net_registrar", "type":["null","string"],"doc":"Registrar", "default": null},
+ {"name":"net_registrar_iana_id", "type":["null","string"],"doc":"Registrar IANA ID", "default": null},
+ {"name":"net_registrar_abuse_contact_email", "type":["null","string"],"doc":"Registrar Abuse Contact Email", "default": null},
+ {"name":"net_registrar_abuse_contact_phone", "type":["null","string"],"doc":"Registrar Abuse Contact Phone", "default": null},
+ {"name":"net_domain_status", "type":["null","string"],"doc":"Domain Status", "default": null},
+ {"name":"net_registry_registrant_id", "type":["null","string"],"doc":"Registry Registrant ID", "default": null},
+ {"name":"net_registrant_name", "type":["null","string"],"doc":"Registrant Name", "default": null},
+ {"name":"net_registrant_organization", "type":["null","string"],"doc":"Registrant Organization", "default": null},
+ {"name":"net_registrant_street", "type":["null","string"],"doc":"Registrant Street", "default": null},
+ {"name":"net_registrant_city", "type":["null","string"],"doc":"Registrant City", "default": null},
+ {"name":"net_registrant_state", "type":["null","string"],"doc":"Registrant State/Province", "default": null},
+ {"name":"net_registrant_post_code", "type":["null","string"],"doc":"Registrant Postal Code", "default": null},
+ {"name":"net_registrant_country", "type":["null","string"],"doc":"Registrant Country", "default": null},
+ {"name":"net_registrant_phone", "type":["null","string"],"doc":"Registrant Phone", "default": null},
+ {"name":"net_registrant_email", "type":["null","string"],"doc":"Registrant Email", "default": null},
+ {"name":"net_registry_admin_id", "type":["null","string"],"doc":"Registry Admin ID", "default": null},
+ {"name":"net_name_servers", "type":["null","string"],"doc":"Name Server", "default": null},
+ {"name":"net_dnssec", "type":["null","string"],"doc":"DNSSEC", "default": null},
+ {"name":"net_risk", "type":["null","float"],"doc":"Risk score", "default": null}
+ ],
+ "doc": "A view schema for storing Apache Spot Network Context data."
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0ee2d06b/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
new file mode 100755
index 0000000..a2d8a51
--- /dev/null
+++ b/spot-setup/odm/odm_setup.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Instructions
+# To execute this script, run ./odm_setup with a format type (pqt, avro) as an argument.
+#
+# i.e. ./odm_setup pqt
+#
+# NOTE: At this time only Parquet and Avro storage formats are supported for the ODM tables.
+
+# Check the format argument and make sure its supported
+format=$1
+if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
+ echo "Format argument '$format' is not supported. Only Parquet and Avro are supported data storage formats. Use 'pqt' or 'avro' instead (i.e. ./odm_setup pqt)."
+ exit 0
+fi
+
+DSOURCES=('odm')
+DFOLDERS=(
+'event'
+'user_context'
+'endpoint_context'
+'network_context'
+'threat_intelligence_context'
+'vulnerability_context'
+)
+
+# Sourcing ODM Spot configuration variables
+source /etc/spot.conf
+
+# Creating HDFS user's folder
+sudo -u hdfs hdfs dfs -mkdir ${HUSER}
+sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
+sudo -u hdfs hdfs dfs -chmod 775 ${HUSER}
+
+# Creating HDFS paths for each use case
+for d in "${DSOURCES[@]}"
+do
+ echo "creating /$d"
+ sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d
+
+ # Create Avro schemas directory on HDFS if Avro storage is selected
+ if [ "$format" == "avro" ] ; then
+ echo "creating /$d/schema"
+ sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d/schema
+ fi
+
+ for f in "${DFOLDERS[@]}"
+ do
+ echo "creating $d/$f"
+ sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d/$f
+ done
+
+ # Modifying permission on HDFS folders to allow Impala to read/write
+ echo "modifying permissions recursively on ${HUSER}/$d"
+ sudo -u hdfs hdfs dfs -chmod -R 775 ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
+done
+
+# Creating Spot Database
+impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+
+# Creating ODM Impala tables
+for d in "${DSOURCES[@]}"
+do
+ for f in "${DFOLDERS[@]}"
+ do
+ #If desired storage format is parquet, create ODM as Parquet tables
+ if [ "$format" == "pqt" ] ; then
+ echo "Creating ODM Impala Parquet table ${f}..."
+ echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
+
+ impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
+ fi
+ # If desired storage format is "avro", create ODM as Avro tables with Avro schemas
+ if [ "$format" == "avro" ] ; then
+ echo "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
+ echo "sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+
+ sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+
+ echo "Creating ODM Impala Avro table ${f}..."
+ echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
+
+ impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
+ fi
+ done
+done
\ No newline at end of file