You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/04/05 16:27:33 UTC
[1/2] incubator-metron git commit: METRON-88: Expose rotation policy
in Flux configuration of HDFSWriter (dlyle65535 via cestella) closes
apache/incubator-metron#63
Repository: incubator-metron
Updated Branches:
refs/heads/master 1d1b9d38c -> edeec014c
METRON-88: Expose rotation policy in Flux configuration of HDFSWriter (dlyle65535 via cestella) closes apache/incubator-metron#63
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/885e1975
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/885e1975
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/885e1975
Branch: refs/heads/master
Commit: 885e1975e216b4b94471cd0d34eee111937aeeae
Parents: ad3866b
Author: dlyle65535 <dl...@gmail.com>
Authored: Tue Apr 5 10:26:38 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Tue Apr 5 10:26:38 2016 -0400
----------------------------------------------------------------------
deployment/roles/metron_streaming/defaults/main.yml | 10 ++++++++++
deployment/roles/metron_streaming/tasks/main.yml | 4 +++-
deployment/roles/metron_streaming/vars/main.yml | 7 +------
.../resources/Metron_Configs/etc/env/config.properties | 3 +++
.../Metron_Configs/topologies/enrichment/remote.yaml | 10 ++++++++++
5 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/885e1975/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
index 3e96705..0dd21e3 100644
--- a/deployment/roles/metron_streaming/defaults/main.yml
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -34,3 +34,13 @@ storm_topologies:
- "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
- "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
+
+elasticsearch_config_path: /etc/elasticsearch
+elasticsearch_cluster_name: metron
+elasticsearch_transport_port: 9300
+hbase_config_path: "/etc/hbase/conf"
+hdfs_config_path: "/etc/hadoop/conf"
+metron_hdfs_output_dir: "/apps/metron"
+metron_hdfs_rotation_policy: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+metron_hdfs_rotation_policy_count: 1
+metron_hdfs_rotation_policy_units: DAYS
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/885e1975/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index c0e0638..55ace91 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -77,7 +77,9 @@
- { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
-
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
- include: source_config.yml
run_once: true
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/885e1975/deployment/roles/metron_streaming/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/vars/main.yml b/deployment/roles/metron_streaming/vars/main.yml
index b9664c9..b356762 100644
--- a/deployment/roles/metron_streaming/vars/main.yml
+++ b/deployment/roles/metron_streaming/vars/main.yml
@@ -24,9 +24,4 @@ metron_dataloads_path: "{{ playbook_dir }}/../../metron-streaming/Metron-DataLoa
metron_topologies_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_topologies_bundle_name }}"
metron_src_config_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs"
metron_properties_config_path: "{{ metron_directory }}/config/etc/env/config.properties"
-elasticsearch_config_path: /etc/elasticsearch
-elasticsearch_cluster_name: metron
-elasticsearch_transport_port: 9300
-hbase_config_path: "/etc/hbase/conf"
-hdfs_config_path: "/etc/hadoop/conf"
-metron_hdfs_output_dir: "/apps/metron"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/885e1975/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
index 5d36ef1..232f2be 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
@@ -87,6 +87,9 @@ bolt.hdfs.wip.file.path=/paloalto/wip
bolt.hdfs.finished.file.path=/paloalto/rotated
bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
index.hdfs.output=/tmp/metron/enriched
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.count=1
+bolt.hdfs.rotation.policy.units=DAYS
##### HBase #####
bolt.hbase.table.name=pcap
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/885e1975/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
index d3961bf..eb2d2d6 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
@@ -113,6 +113,12 @@ components:
- name: "withPath"
args:
- "${index.hdfs.output}"
+
+ - id: "hdfsRotationPolicy"
+ className: "${bolt.hdfs.rotation.policy}"
+ constructorArgs:
+ - ${bolt.hdfs.rotation.policy.count}
+ - "${bolt.hdfs.rotation.policy.units}"
#indexing
- id: "hdfsWriter"
className: "org.apache.metron.writer.hdfs.HdfsWriter"
@@ -120,6 +126,10 @@ components:
- name: "withFileNameFormat"
args:
- ref: "fileNameFormat"
+ - name: "withRotationPolicy"
+ args:
+ - ref: "hdfsRotationPolicy"
+
- id: "indexWriter"
className: "org.apache.metron.writer.ElasticsearchWriter"
constructorArgs:
[2/2] incubator-metron git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-metron
Posted by ce...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-metron
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/edeec014
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/edeec014
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/edeec014
Branch: refs/heads/master
Commit: edeec014c21b61b7efd2955f76843b58119cbaa5
Parents: 885e197 1d1b9d3
Author: cstella <ce...@gmail.com>
Authored: Tue Apr 5 10:27:00 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Tue Apr 5 10:27:00 2016 -0400
----------------------------------------------------------------------
deployment/roles/metron_streaming/tasks/grok_upload.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------