You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by le...@apache.org on 2017/04/17 16:37:33 UTC
[04/16] incubator-spot git commit: HDFS validation when the scored
connections are reset
HDFS validation when the scored connections are reset
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/66489111
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/66489111
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/66489111
Branch: refs/heads/SPOT-35_graphql_api
Commit: 664891113dcfc05d8947ac3bc176adc407ef6a0c
Parents: 2b67248
Author: Everardo Lopez Sandoval (Intel) <el...@elopezsa-mac02.zpn.intel.com>
Authored: Fri Apr 7 17:47:41 2017 -0500
Committer: Everardo Lopez Sandoval (Intel) <el...@elopezsa-mac02.zpn.intel.com>
Committed: Fri Apr 7 17:47:41 2017 -0500
----------------------------------------------------------------------
spot-oa/api/resources/dns.py | 22 ++++++++++++++--------
spot-oa/api/resources/flow.py | 22 ++++++++++++++--------
spot-oa/api/resources/proxy.py | 22 ++++++++++++++--------
3 files changed, 42 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/66489111/spot-oa/api/resources/dns.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/dns.py b/spot-oa/api/resources/dns.py
index 0758812..f59ceae 100644
--- a/spot-oa/api/resources/dns.py
+++ b/spot-oa/api/resources/dns.py
@@ -1,6 +1,7 @@
import api.resources.impala_engine as ImpalaEngine
import api.resources.hdfs_client as HDFSClient
import api.resources.configurator as Configuration
+from hdfs.util import HdfsError
"""
--------------------------------------------------------------------------
@@ -361,11 +362,16 @@ def reset_scored_connections(date):
dns_timeline = "dns/hive/oa/threat_investigation"
app_path = Configuration.spot()
- # remove parquet files manually to allow the comments update.
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,dns_storyboard,date.year,date.month,date.day) , "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,dns_threat_investigation,date.year,date.month,date.day), "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,dns_timeline,date.year,date.month,date.day), "impala")
- ImpalaEngine.execute_query("invalidate metadata")
\ No newline at end of file
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/66489111/spot-oa/api/resources/flow.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/flow.py b/spot-oa/api/resources/flow.py
index c4791df..cf5b284 100755
--- a/spot-oa/api/resources/flow.py
+++ b/spot-oa/api/resources/flow.py
@@ -3,6 +3,7 @@ import api.resources.hdfs_client as HDFSClient
import api.resources.configurator as Configuration
import os
import struct, socket
+from hdfs.util import HdfsError
import json
"""
@@ -921,11 +922,16 @@ def reset_scored_connections(date):
flow_timeline = "flow/hive/oa/timeline"
app_path = Configuration.spot()
- # remove parquet files manually to allow the comments update.
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,flow_storyboard,date.year,date.month,date.day) , "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,flow_threat_investigation,date.year,date.month,date.day), "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,flow_timeline,date.year,date.month,date.day), "impala")
- ImpalaEngine.execute_query("invalidate metadata")
\ No newline at end of file
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/66489111/spot-oa/api/resources/proxy.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/proxy.py b/spot-oa/api/resources/proxy.py
index 37a2828..b9a6052 100644
--- a/spot-oa/api/resources/proxy.py
+++ b/spot-oa/api/resources/proxy.py
@@ -1,6 +1,7 @@
import md5
import api.resources.impala_engine as ImpalaEngine
import api.resources.hdfs_client as HDFSClient
+from hdfs.util import HdfsError
import api.resources.configurator as Configuration
from collections import defaultdict
import json
@@ -418,11 +419,16 @@ def reset_scored_connections(date):
proxy_timeline = "proxy/hive/oa/threat_investigation"
app_path = Configuration.spot()
- # remove parquet files manually to allow the comments update.
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,proxy_storyboard,date.year,date.month,date.day) , "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,proxy_threat_investigation,date.year,date.month,date.day), "impala")
- HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
- app_path,proxy_timeline,date.year,date.month,date.day), "impala")
- ImpalaEngine.execute_query("invalidate metadata")
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
\ No newline at end of file