You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/08/17 17:07:49 UTC
[2/2] incubator-eagle git commit: EAGLE-470 data enrichment refactor
in eagle security monitoring data enrichment refactor in eagle security
monitoring https://issues.apache.org/jira/browse/EAGLE-470
EAGLE-470 data enrichment refactor in eagle security monitoring
data enrichment refactor in eagle security monitoring
https://issues.apache.org/jira/browse/EAGLE-470
Author: @yonzhang <yo...@gmail.com>
Closes: #358
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/92ca7c54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/92ca7c54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/92ca7c54
Branch: refs/heads/develop
Commit: 92ca7c54233a71689da8cffd01e282ee8f8de05e
Parents: 48bb082
Author: yonzhang <yo...@gmail.com>
Authored: Wed Aug 17 10:02:41 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Wed Aug 17 10:02:41 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/log4j.properties | 4 +-
.../security/enrich/AbstractDataEnrichBolt.java | 67 ++++++++
.../security/enrich/AbstractDataEnrichLCM.java | 51 ++++++
.../eagle/security/enrich/DataEnrichJob.java | 55 +++++++
.../eagle/security/enrich/DataEnrichLCM.java | 45 ++++++
.../security/enrich/ExternalDataCache.java | 41 +++++
.../security/enrich/ExternalDataJoiner.java | 99 ++++++++++++
.../entity/FileSensitivityAPIEntity.java | 52 ------
.../HbaseResourceSensitivityAPIEntity.java | 47 ------
.../HiveResourceSensitivityAPIEntity.java | 53 ------
.../eagle/security/entity/IPZoneEntity.java | 52 ------
.../entity/SecurityEntityRepository.java | 5 -
.../service/IMetadataServiceClient.java | 38 -----
.../ISecurityDataEnrichServiceClient.java | 38 +++++
.../service/MetadataServiceClientImpl.java | 160 -------------------
.../SecurityDataEnrichServiceClientImpl.java | 160 +++++++++++++++++++
.../AbstractResourceSensitivityPollingJob.java | 56 -------
.../eagle/security/util/ExternalDataCache.java | 43 -----
.../eagle/security/util/ExternalDataJoiner.java | 109 -------------
.../hbase/HBaseSensitivityDataEnrichLCM.java | 43 +++++
.../HbaseResourceSensitivityDataJoinBolt.java | 41 +----
.../HbaseResourceSensitivityPollingJob.java | 72 ---------
.../AbstractHdfsAuditLogApplication.java | 4 +-
.../auditlog/FileSensitivityDataJoinBolt.java | 108 -------------
.../auditlog/HdfsSensitivityDataEnrichBolt.java | 82 ++++++++++
.../auditlog/HdfsSensitivityDataEnrichLCM.java | 43 +++++
.../security/auditlog/IPZoneDataEnrichBolt.java | 62 +++++++
.../security/auditlog/IPZoneDataEnrichLCM.java | 43 +++++
.../security/auditlog/IPZoneDataJoinBolt.java | 83 ----------
.../timer/FileSensitivityPollingJob.java | 74 ---------
.../auditlog/timer/IPZonePollingJob.java | 77 ---------
.../hdfs/HDFSResourceSensitivityDataJoiner.java | 8 +-
.../hdfs/HDFSResourceSensitivityService.java | 143 ++++++++---------
.../hdfs/rest/HDFSResourceWebResource.java | 3 +-
.../hdfs/test/HDFSChildSensitivityTypeTest.java | 8 +-
.../hive/HiveQueryMonitoringApplication.java | 4 +-
.../HiveResourceSensitivityDataJoinBolt.java | 105 ------------
.../HiveResourceSensitivityPollingJob.java | 83 ----------
.../HiveSensitivityDataEnrichBolt.java | 82 ++++++++++
.../HiveSensitivityDataEnrichLCM.java | 43 +++++
.../hdfs/rest/MAPRFSResourceWebResource.java | 4 +-
.../OozieResourceSensitivityDataJoinBolt.java | 34 +---
.../OozieResourceSensitivityPollingJob.java | 63 --------
.../OozieSensitivityDataEnrichLCM.java | 42 +++++
44 files changed, 1094 insertions(+), 1435 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties b/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
index 6c31ebc..f6076a6 100644
--- a/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
+++ b/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
@@ -19,8 +19,8 @@ eagle.log.dir=./logs
eagle.log.file=eagle.log
-#log4j.logger.eagle.security.auditlog.IPZoneDataJoinBolt=DEBUG
-#log4j.logger.eagle.security.auditlog.FileSensitivityDataJoinBolt=DEBUG
+#log4j.logger.eagle.security.auditlog.IPZoneDataEnrichBolt=DEBUG
+#log4j.logger.eagle.security.auditlog.HdfsSensitivityDataEnrichBolt=DEBUG
#log4j.logger.eagle.executor.AlertExecutor=DEBUG
# standard output
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
new file mode 100644
index 0000000..6c23180
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ */
+public abstract class AbstractDataEnrichBolt<T, K> extends BaseRichBolt {
+ private final static Logger LOG = LoggerFactory.getLogger(AbstractDataEnrichBolt.class);
+ protected Config config;
+ protected OutputCollector collector;
+ private DataEnrichLCM<T, K> lcm;
+
+ public AbstractDataEnrichBolt(Config config, DataEnrichLCM<T, K> lcm){
+ this.config = config;
+ this.lcm = lcm;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ // start external data retrieval
+ try {
+ ExternalDataJoiner joiner = new ExternalDataJoiner(
+ lcm, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
+ joiner.start();
+ } catch(Exception ex){
+ LOG.error("Fail bringing up quartz scheduler.", ex);
+ throw new IllegalStateException(ex);
+ }
+ }
+
+ protected abstract void executeWithEnrich(Tuple input, Map<K, T> cache);
+
+ @Override
+ public void execute(Tuple input) {
+ Map map = (Map) ExternalDataCache
+ .getInstance()
+ .getJobResult(lcm.getClass());
+ executeWithEnrich(input, map);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
new file mode 100644
index 0000000..002bf82
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.service.SecurityDataEnrichServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public abstract class AbstractDataEnrichLCM<T, K> implements DataEnrichLCM<T, K> {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractDataEnrichLCM.class);
+ private Config config;
+ public AbstractDataEnrichLCM(Config config){
+ this.config = config;
+ }
+
+ @Override
+ public Collection<T> loadExternal() {
+ String eagleServiceHost = config.getString("eagleService.host");
+ Integer eagleServicePort = config.getInt("eagleService.port");
+
+ // load from eagle database
+ LOG.info("Load sensitivity information from eagle service " + eagleServiceHost + ":" + eagleServicePort);
+
+ ISecurityDataEnrichServiceClient client = new SecurityDataEnrichServiceClientImpl(eagleServiceHost, eagleServicePort, "/rest");
+ return loadFromService(client);
+ }
+
+ protected abstract Collection<T> loadFromService(ISecurityDataEnrichServiceClient client);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
new file mode 100644
index 0000000..13777bf
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.google.common.collect.Maps;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ */
+public class DataEnrichJob implements Job {
+ private final static Logger LOG = LoggerFactory.getLogger(DataEnrichJob.class);
+
+ @Override
+ public void execute(JobExecutionContext context) throws JobExecutionException {
+ JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
+
+ DataEnrichLCM lcm = (DataEnrichLCM)jobDataMap.getOrDefault("dataEnrichLCM", null);
+ if(lcm == null)
+ throw new IllegalStateException("dataEnrichLCM implementation should be provided");
+ try {
+ Collection externalEntities = lcm.loadExternal();
+ Map<Object, Object> map = Maps.uniqueIndex(
+ externalEntities,
+ entity -> lcm.getCacheKey(entity)
+ );
+ ExternalDataCache.getInstance().setJobResult(lcm.getClass(), map);
+ } catch(Exception ex) {
+ LOG.error("Fail to load hbase resource sensitivity data", ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
new file mode 100644
index 0000000..58fecf1
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ * Data enrichment lifecycle methods
+ * @param <T> entity type, this entity will be put into cache
+ * @param <K> cache key
+ */
+public interface DataEnrichLCM<T, K> extends Serializable{
+ /**
+ * load all external data for real time enrichment
+ *
+ * @return
+ */
+ Collection<T> loadExternal();
+
+ /**
+ * get cache key from one entity
+ *
+ * @param entity
+ * @return
+ */
+ K getCacheKey(T entity);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
new file mode 100644
index 0000000..0293b30
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.enrich;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ExternalDataCache {
+ private Map<Class<? extends DataEnrichLCM>, Object> map = new ConcurrentHashMap<>();
+
+ private static final ExternalDataCache instance = new ExternalDataCache();
+
+ private ExternalDataCache(){
+ }
+
+ public static ExternalDataCache getInstance(){
+ return instance;
+ }
+
+ public Object getJobResult(Class<? extends DataEnrichLCM> cls){
+ return map.get(cls);
+ }
+
+ public void setJobResult(Class<? extends DataEnrichLCM> cls, Object jobResult){
+ map.put(cls, jobResult);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
new file mode 100644
index 0000000..1f593a8
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.util.JVMSingleQuartzScheduler;
+import org.quartz.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * spawn Quartz scheduler to poll data from eagle service and invoke custom job to handle
+ */
+public class ExternalDataJoiner {
+ private static final Logger LOG = LoggerFactory.getLogger(ExternalDataJoiner.class);
+
+ private Scheduler sched;
+ private JobDataMap jobDataMap;
+ private String jobId;
+ private String taskId;
+ private Config config;
+
+ private static final String DATA_JOIN_POLL_INTERVALSEC = "dataJoinPollIntervalSec";
+ private static final String QUARTZ_GROUP_NAME = "dataJoiner";
+
+ private final int defaultIntervalSeconds = 60;
+
+ /**
+ * constructor with parameters which are passed from customer
+ * throws Exception while this scheduler can't be brought up
+ * Some protocols:
+ * 1. prop should include an entry keyed by "data.join.poll.intervalInSeconds" if you want to customize poll interval.
+ */
+ public ExternalDataJoiner(DataEnrichLCM lcm, Config config, String taskId) throws Exception{
+ this.taskId = taskId;
+ Map<String, Object> map = new HashMap<>();
+ // add DataEnrichLCM to JobDataMap
+ map.put("dataEnrichLCM", lcm);
+ this.config = config;
+ init(lcm.getClass().getCanonicalName(), map);
+ }
+
+ private void init(String jobId, Map<String, Object> prop) throws Exception{
+ this.jobId = jobId;
+ jobDataMap = new JobDataMap(prop);
+ sched = JVMSingleQuartzScheduler.getInstance().getScheduler();
+ }
+
+ public void start(){
+ // for job
+ String group = String.format("%s.%s.%s.%s",
+ QUARTZ_GROUP_NAME,
+ config.hasPath("siteId") ? config.getString("siteId") : "",
+ config.hasPath("appId") ? config.getString("appId") : "",
+ taskId);
+ JobDetail job = JobBuilder.newJob(DataEnrichJob.class)
+ .withIdentity(jobId + ".job", group)
+ .usingJobData(jobDataMap)
+ .build();
+
+ // for trigger
+ Object interval = jobDataMap.get(DATA_JOIN_POLL_INTERVALSEC);
+ int dataJoinPollIntervalSec = (interval == null ? defaultIntervalSeconds : Integer.parseInt(interval.toString()));
+ Trigger trigger = TriggerBuilder.newTrigger()
+ .withIdentity(jobId + ".trigger", group)
+ .startNow()
+ .withSchedule(SimpleScheduleBuilder.simpleSchedule()
+ .withIntervalInSeconds(dataJoinPollIntervalSec)
+ .repeatForever())
+ .build();
+ try{
+ sched.scheduleJob(job, trigger);
+ }catch(Exception ex){
+ LOG.error("Can't schedule job " + job.getDescription(), ex);
+ }
+ }
+
+ public void stop() throws Exception{
+ sched.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
deleted file mode 100644
index 9eef192..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("fileSensitivity")
-@ColumnFamily("f")
-@Prefix("fileSensitivity")
-@Service("FileSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "filedir"})
-public class FileSensitivityAPIEntity extends TaggedLogAPIEntity{
- /**
- * sensitivityType can be multi-value attribute, and values can be separated by "|"
- */
- @Column("a")
- private String sensitivityType;
-
- public String getSensitivityType() {
- return sensitivityType;
- }
-
- public void setSensitivityType(String sensitivityType) {
- this.sensitivityType = sensitivityType;
- valueChanged("sensitivityType");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
deleted file mode 100644
index 551967d..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.security.entity;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("hbaseResourceSensitivity")
-@ColumnFamily("f")
-@Prefix("hbaseResourceSensitivity")
-@Service("HbaseResourceSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "hbaseResource"})
-public class HbaseResourceSensitivityAPIEntity extends TaggedLogAPIEntity {
- private static final long serialVersionUID = 2L;
- /**
- * sensitivityType can be multi-value attribute, and values can be separated by "|"
- */
- @Column("a")
- private String sensitivityType;
-
- public String getSensitivityType() {
- return sensitivityType;
- }
-
- public void setSensitivityType(String sensitivityType) {
- this.sensitivityType = sensitivityType;
- valueChanged("sensitivityType");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
deleted file mode 100644
index aebec8b..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("hiveResourceSensitivity")
-@ColumnFamily("f")
-@Prefix("hiveResourceSensitivity")
-@Service("HiveResourceSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "hiveResource"})
-public class HiveResourceSensitivityAPIEntity extends TaggedLogAPIEntity {
- private static final long serialVersionUID = 1L;
- /**
- * sensitivityType can be multi-value attribute, and values can be separated by "|"
- */
- @Column("a")
- private String sensitivityType;
-
- public String getSensitivityType() {
- return sensitivityType;
- }
-
- public void setSensitivityType(String sensitivityType) {
- this.sensitivityType = sensitivityType;
- valueChanged("sensitivityType");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
deleted file mode 100644
index f944ace..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-/**
- * ipranger can be single IP, host or ip with subnet etc.
- */
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("ipzone")
-@ColumnFamily("f")
-@Prefix("ipzone")
-@Service("IPZoneService")
-@TimeSeries(false)
-@Tags({"iphost"})
-public class IPZoneEntity extends TaggedLogAPIEntity{
- @Column("a")
- private String securityZone;
-
- public String getSecurityZone() {
- return securityZone;
- }
-
- public void setSecurityZone(String securityZone) {
- this.securityZone = securityZone;
- valueChanged("securityZone");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
index 5d63690..7c51724 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
@@ -23,11 +23,6 @@ import org.apache.eagle.log.entity.repo.EntityRepository;
public class SecurityEntityRepository extends EntityRepository {
public SecurityEntityRepository() {
- entitySet.add(HbaseResourceSensitivityAPIEntity.class);
- entitySet.add(FileSensitivityAPIEntity.class);
- entitySet.add(IPZoneEntity.class);
entitySet.add(HdfsUserCommandPatternEntity.class);
- entitySet.add(HiveResourceSensitivityAPIEntity.class);
- entitySet.add(OozieResourceSensitivityAPIEntity.class);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
deleted file mode 100644
index d87cf13..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * * <p/>
- * * http://www.apache.org/licenses/LICENSE-2.0
- * * <p/>
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.security.service;
-
-import java.io.Closeable;
-import java.io.Serializable;
-import java.util.Collection;
-
-/**
- * service stub to get metadata from remote metadata service
- */
-public interface IMetadataServiceClient extends Closeable, Serializable {
- Collection<HBaseSensitivityEntity> listHBaseSensitivities();
- OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h);
- Collection<HdfsSensitivityEntity> listHdfsSensitivities();
- OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h);
- Collection<IPZoneEntity> listIPZones();
- OpResult addIPZone(Collection<IPZoneEntity> h);
- Collection<HiveSensitivityEntity> listHiveSensitivities();
- OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h);
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
new file mode 100644
index 0000000..916b08f
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * * <p/>
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * * <p/>
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.service;
+
+import java.io.Closeable;
+import java.io.Serializable;
+import java.util.Collection;
+
+/**
+ * service stub to get metadata from remote metadata service
+ */
+public interface ISecurityDataEnrichServiceClient extends Closeable, Serializable {
+ Collection<HBaseSensitivityEntity> listHBaseSensitivities();
+ OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h);
+ Collection<HdfsSensitivityEntity> listHdfsSensitivities();
+ OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h);
+ Collection<IPZoneEntity> listIPZones();
+ OpResult addIPZone(Collection<IPZoneEntity> h);
+ Collection<HiveSensitivityEntity> listHiveSensitivities();
+ OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
deleted file mode 100644
index c350c2c..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * * <p/>
- * * http://www.apache.org/licenses/LICENSE-2.0
- * * <p/>
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-
-package org.apache.eagle.security.service;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import com.typesafe.config.Config;
-import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-public class MetadataServiceClientImpl implements IMetadataServiceClient {
- private static final long serialVersionUID = 3003976065082684128L;
-
- private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClientImpl.class);
-
- private static final String METADATA_LIST_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
- private static final String METADATA_ADD_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
-
- private static final String METADATA_LIST_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
- private static final String METADATA_ADD_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
-
- private static final String METADATA_LIST_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
- private static final String METADATA_ADD_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
-
- private static final String METADATA_LIST_IPZONE_PATH = "/metadata/security/ipzone";
- private static final String METADATA_ADD_IPZONE_PATH = "/metadata/security/ipzone";
-
- private static final String EAGLE_CORRELATION_CONTEXT = "metadataService.context";
- private static final String EAGLE_CORRELATION_SERVICE_PORT = "metadataService.port";
- private static final String EAGLE_CORRELATION_SERVICE_HOST = "metadataService.host";
-
- protected static final String CONTENT_TYPE = "Content-Type";
-
- private String host;
- private int port;
- private String context;
- private transient Client client;
- private String basePath;
-
- public MetadataServiceClientImpl(Config config) {
- this(config.getString(EAGLE_CORRELATION_SERVICE_HOST), config.getInt(EAGLE_CORRELATION_SERVICE_PORT), config
- .getString(EAGLE_CORRELATION_CONTEXT));
- basePath = buildBasePath();
- }
-
- public MetadataServiceClientImpl(String host, int port, String context) {
- this.host = host;
- this.port = port;
- this.context = context;
- this.basePath = buildBasePath();
- ClientConfig cc = new DefaultClientConfig();
- cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
- cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
- cc.getClasses().add(JacksonJsonProvider.class);
- cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
- this.client = Client.create(cc);
- client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
- }
-
- private String buildBasePath() {
- StringBuilder sb = new StringBuilder();
- sb.append("http://");
- sb.append(host);
- sb.append(":");
- sb.append(port);
- sb.append(context);
- return sb.toString();
- }
-
- private <T> List<T> list(String path, GenericType<List<T>> type) {
- WebResource r = client.resource(basePath + path);
- LOG.info("query URL {}", basePath + path);
- List<T> ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type);
- return ret;
- }
-
- @Override
- public void close() throws IOException {
- client.destroy();
- }
-
- @Override
- public Collection<HBaseSensitivityEntity> listHBaseSensitivities() {
- return list(METADATA_LIST_HBASE_SENSITIVITY_PATH, new GenericType<List<HBaseSensitivityEntity>>() {
- });
- }
-
- @Override
- public OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) {
- WebResource r = client.resource(basePath + METADATA_ADD_HBASE_SENSITIVITY_PATH);
- r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
- return new OpResult();
- }
-
- @Override
- public Collection<HdfsSensitivityEntity> listHdfsSensitivities() {
- return list(METADATA_LIST_HDFS_SENSITIVITY_PATH, new GenericType<List<HdfsSensitivityEntity>>() {
- });
- }
-
- @Override
- public OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h) {
- WebResource r = client.resource(basePath + METADATA_ADD_HDFS_SENSITIVITY_PATH);
- r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
- return new OpResult();
- }
-
- @Override
- public Collection<IPZoneEntity> listIPZones(){
- return list(METADATA_LIST_IPZONE_PATH, new GenericType<List<IPZoneEntity>>() {
- });
- }
-
- @Override
- public OpResult addIPZone(Collection<IPZoneEntity> h){
- WebResource r = client.resource(basePath + METADATA_ADD_IPZONE_PATH);
- r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
- return new OpResult();
- }
-
- @Override
- public Collection<HiveSensitivityEntity> listHiveSensitivities() {
- return list(METADATA_LIST_HIVE_SENSITIVITY_PATH, new GenericType<List<HiveSensitivityEntity>>() {
- });
- }
-
- @Override
- public OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h) {
- WebResource r = client.resource(basePath + METADATA_ADD_HIVE_SENSITIVITY_PATH);
- r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
- return new OpResult();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
new file mode 100644
index 0000000..ffabcbc
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
@@ -0,0 +1,160 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * * <p/>
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * * <p/>
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.service;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.typesafe.config.Config;
+import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+public class SecurityDataEnrichServiceClientImpl implements ISecurityDataEnrichServiceClient {
+ private static final long serialVersionUID = 3003976065082684128L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(SecurityDataEnrichServiceClientImpl.class);
+
+ private static final String METADATA_LIST_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
+ private static final String METADATA_ADD_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
+
+ private static final String METADATA_LIST_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
+ private static final String METADATA_ADD_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
+
+ private static final String METADATA_LIST_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
+ private static final String METADATA_ADD_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
+
+ private static final String METADATA_LIST_IPZONE_PATH = "/metadata/security/ipzone";
+ private static final String METADATA_ADD_IPZONE_PATH = "/metadata/security/ipzone";
+
+ private static final String EAGLE_CORRELATION_CONTEXT = "metadataService.context";
+ private static final String EAGLE_CORRELATION_SERVICE_PORT = "metadataService.port";
+ private static final String EAGLE_CORRELATION_SERVICE_HOST = "metadataService.host";
+
+ protected static final String CONTENT_TYPE = "Content-Type";
+
+ private String host;
+ private int port;
+ private String context;
+ private transient Client client;
+ private String basePath;
+
+ public SecurityDataEnrichServiceClientImpl(Config config) {
+ this(config.getString(EAGLE_CORRELATION_SERVICE_HOST), config.getInt(EAGLE_CORRELATION_SERVICE_PORT), config
+ .getString(EAGLE_CORRELATION_CONTEXT));
+ basePath = buildBasePath();
+ }
+
+ public SecurityDataEnrichServiceClientImpl(String host, int port, String context) {
+ this.host = host;
+ this.port = port;
+ this.context = context;
+ this.basePath = buildBasePath();
+ ClientConfig cc = new DefaultClientConfig();
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
+ cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
+ cc.getClasses().add(JacksonJsonProvider.class);
+ cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
+ this.client = Client.create(cc);
+ client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
+ }
+
+ private String buildBasePath() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("http://");
+ sb.append(host);
+ sb.append(":");
+ sb.append(port);
+ sb.append(context);
+ return sb.toString();
+ }
+
+ private <T> List<T> list(String path, GenericType<List<T>> type) {
+ WebResource r = client.resource(basePath + path);
+ LOG.info("query URL {}", basePath + path);
+ List<T> ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type);
+ return ret;
+ }
+
+ @Override
+ public void close() throws IOException {
+ client.destroy();
+ }
+
+ @Override
+ public Collection<HBaseSensitivityEntity> listHBaseSensitivities() {
+ return list(METADATA_LIST_HBASE_SENSITIVITY_PATH, new GenericType<List<HBaseSensitivityEntity>>() {
+ });
+ }
+
+ @Override
+ public OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) {
+ WebResource r = client.resource(basePath + METADATA_ADD_HBASE_SENSITIVITY_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+ return new OpResult();
+ }
+
+ @Override
+ public Collection<HdfsSensitivityEntity> listHdfsSensitivities() {
+ return list(METADATA_LIST_HDFS_SENSITIVITY_PATH, new GenericType<List<HdfsSensitivityEntity>>() {
+ });
+ }
+
+ @Override
+ public OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h) {
+ WebResource r = client.resource(basePath + METADATA_ADD_HDFS_SENSITIVITY_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+ return new OpResult();
+ }
+
+ @Override
+ public Collection<IPZoneEntity> listIPZones(){
+ return list(METADATA_LIST_IPZONE_PATH, new GenericType<List<IPZoneEntity>>() {
+ });
+ }
+
+ @Override
+ public OpResult addIPZone(Collection<IPZoneEntity> h){
+ WebResource r = client.resource(basePath + METADATA_ADD_IPZONE_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+ return new OpResult();
+ }
+
+ @Override
+ public Collection<HiveSensitivityEntity> listHiveSensitivities() {
+ return list(METADATA_LIST_HIVE_SENSITIVITY_PATH, new GenericType<List<HiveSensitivityEntity>>() {
+ });
+ }
+
+ @Override
+ public OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h) {
+ WebResource r = client.resource(basePath + METADATA_ADD_HIVE_SENSITIVITY_PATH);
+ r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+ return new OpResult();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
deleted file mode 100644
index 8cd1317..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.security.util;
-
-
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.client.IEagleServiceClient;
-import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
-import org.quartz.JobDataMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-public class AbstractResourceSensitivityPollingJob {
- private final static Logger LOG = LoggerFactory.getLogger(AbstractResourceSensitivityPollingJob.class);
-
- protected <R extends TaggedLogAPIEntity> List<R> load(JobDataMap jobDataMap, String service) throws Exception {
- Map<String, Object> map = (Map<String, Object>) jobDataMap.get(EagleConfigConstants.EAGLE_SERVICE);
- String eagleServiceHost = (String) map.get(EagleConfigConstants.HOST);
- Integer eagleServicePort = Integer.parseInt(map.get(EagleConfigConstants.PORT).toString());
- String username = map.containsKey(EagleConfigConstants.USERNAME) ? (String) map.get(EagleConfigConstants.USERNAME) : null;
- String password = map.containsKey(EagleConfigConstants.PASSWORD) ? (String) map.get(EagleConfigConstants.PASSWORD) : null;
-
- // load from eagle database
- LOG.info("Load resource sensitivity information from eagle service "
- + eagleServiceHost + ":" + eagleServicePort);
-
- IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, username, password);
- String query = service + "[]{*}";
- GenericServiceAPIResponseEntity<R> response =
- client.search().pageSize(Integer.MAX_VALUE).query(query).send();
- client.close();
- if (response.getException() != null)
- throw new IllegalStateException(response.getException());
- return response.getObj();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
deleted file mode 100644
index 6b7cda4..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.util;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.quartz.Job;
-
-public class ExternalDataCache {
- private Map<Class<? extends Job>, Object> map = new ConcurrentHashMap<Class<? extends Job>, Object>();
-
- private static final ExternalDataCache instance = new ExternalDataCache();
-
- private ExternalDataCache(){
- }
-
- public static ExternalDataCache getInstance(){
- return instance;
- }
-
- public Object getJobResult(Class<? extends Job> cls){
- return map.get(cls);
- }
-
- public void setJobResult(Class<? extends Job> cls, Object jobResult){
- map.put(cls, jobResult);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
deleted file mode 100644
index e6af377..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.util;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.quartz.Job;
-import org.quartz.JobBuilder;
-import org.quartz.JobDataMap;
-import org.quartz.JobDetail;
-import org.quartz.Scheduler;
-import org.quartz.SimpleScheduleBuilder;
-import org.quartz.Trigger;
-import org.quartz.TriggerBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigValue;
-
-/**
- * spawn Quartz scheduler to poll data from eagle service and invoke custom job to handle
- */
-public class ExternalDataJoiner {
- private static final Logger LOG = LoggerFactory.getLogger(ExternalDataJoiner.class);
-
- private Scheduler sched;
- private JobDataMap jobDataMap;
- private Class<? extends Job> jobCls;
- private String id;
- private final static String SCHEDULER_NAME = "OuterDataJoiner.scheduler";
-
- private static final String DATA_JOIN_POLL_INTERVALSEC = "dataJoinPollIntervalSec";
- private static final String QUARTZ_GROUP_NAME = "dataJoiner";
-
- private final int defaultIntervalSeconds = 60;
-
- /**
- * constructor with parameters which are passed from customer
- * throws Exception while this scheduler can't be brought up
- * Some protocols:
- * 1. prop should include an entry keyed by "data.join.poll.intervalInSeconds" if you want to customize poll interval.
- * @param prop
- */
- public ExternalDataJoiner(Class<? extends Job> jobCls, Map<String, Object> prop) throws Exception{
- init(jobCls, prop);
- }
-
- public ExternalDataJoiner(Class<? extends Job> jobCls, Config config, String id) throws Exception{
- this.id = id;
- Map<String, Object> map = new HashMap<>();
- for(Map.Entry<String, ConfigValue> entry : config.getObject("eagleService").entrySet()){
- map.put(entry.getKey(), entry.getValue().unwrapped());
- }
-
- init(jobCls, map);
- }
-
- private void init(Class<? extends Job> jobCls, Map<String, Object> prop) throws Exception{
- this.jobCls = jobCls;
- jobDataMap = new JobDataMap(prop);
- sched = JVMSingleQuartzScheduler.getInstance().getScheduler();
- }
-
- public void start(){
- // for job
- String group = String.format("%s.%s.%s.%s", QUARTZ_GROUP_NAME, jobDataMap.getString(EagleConfigConstants.SITE), jobDataMap.getString(EagleConfigConstants.APPLICATION), id);
- JobDetail job = JobBuilder.newJob(jobCls)
- .withIdentity(jobCls.getName() + ".job", group)
- .usingJobData(jobDataMap)
- .build();
-
- // for trigger
- Object interval = jobDataMap.get(DATA_JOIN_POLL_INTERVALSEC);
- int dataJoinPollIntervalSec = (interval == null ? defaultIntervalSeconds : Integer.parseInt(interval.toString()));
- Trigger trigger = TriggerBuilder.newTrigger()
- .withIdentity(jobCls.getName() + ".trigger", group)
- .startNow()
- .withSchedule(SimpleScheduleBuilder.simpleSchedule()
- .withIntervalInSeconds(dataJoinPollIntervalSec)
- .repeatForever())
- .build();
- try{
- sched.scheduleJob(job, trigger);
- }catch(Exception ex){
- LOG.error("Can't schedule job " + job.getDescription(), ex);
- }
- }
-
- public void stop() throws Exception{
- sched.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
new file mode 100644
index 0000000..2cb1dec
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.hbase;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.enrich.AbstractDataEnrichLCM;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public class HBaseSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HBaseSensitivityEntity, String> {
+ public HBaseSensitivityDataEnrichLCM(Config config){
+ super(config);
+ }
+ @Override
+ protected Collection<HBaseSensitivityEntity> loadFromService(ISecurityDataEnrichServiceClient client) {
+ return client.listHBaseSensitivities();
+ }
+
+ @Override
+ public String getCacheKey(HBaseSensitivityEntity entity) {
+ return entity.getHbaseResource();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
index cb2fa38..c1005cd 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
@@ -6,27 +6,22 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
package org.apache.eagle.security.hbase;
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import com.typesafe.config.Config;
-import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.apache.eagle.security.util.ExternalDataJoiner;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,44 +30,22 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Pattern;
-public class HbaseResourceSensitivityDataJoinBolt extends BaseRichBolt {
+public class HbaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String> {
private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityDataJoinBolt.class);
- private Config config;
- private OutputCollector collector;
public HbaseResourceSensitivityDataJoinBolt(Config config){
- this.config = config;
- }
-
- @Override
- public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- // start hbase sensitivity data polling
- try {
- ExternalDataJoiner joiner = new ExternalDataJoiner(
- HbaseResourceSensitivityPollingJob.class, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
- joiner.start();
- } catch(Exception ex){
- LOG.error("Fail bringing up quartz scheduler.", ex);
- throw new IllegalStateException(ex);
- }
+ super(config, new HBaseSensitivityDataEnrichLCM(config));
}
@Override
- public void execute(Tuple input) {
+ public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) {
try {
- @SuppressWarnings("unchecked")
Map<String, Object> event = (Map<String, Object>) input.getValue(0);
- @SuppressWarnings("unchecked")
- Map<String, HbaseResourceSensitivityAPIEntity> map =
- (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
- .getInstance()
- .getJobResult(HbaseResourceSensitivityPollingJob.class);
LOG.info(">>>> event: " + event + " >>>> map: " + map);
String resource = (String) event.get("scope");
- HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
+ HBaseSensitivityEntity sensitivityEntity = null;
if (map != null && resource != "") {
for (String key : map.keySet()) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
deleted file mode 100644
index a04f1b2..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Maps;
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.apache.eagle.security.service.*;
-import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.quartz.Job;
-import org.quartz.JobDataMap;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.Map;
-
-public class HbaseResourceSensitivityPollingJob extends AbstractResourceSensitivityPollingJob implements Job {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityPollingJob.class);
-
- @Override
- public void execute(JobExecutionContext context)
- throws JobExecutionException {
- JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
- try {
- Collection<HBaseSensitivityEntity> sensitivityEntities = load(jobDataMap);
- Map<String, HBaseSensitivityEntity> map = Maps.uniqueIndex(
- sensitivityEntities,
- new Function<HBaseSensitivityEntity, String>() {
- @Override
- public String apply(HBaseSensitivityEntity input) {
- return input.getHbaseResource();
- }
- });
- ExternalDataCache.getInstance().setJobResult(getClass(), map);
- } catch(Exception ex) {
- LOG.error("Fail to load hbase resource sensitivity data", ex);
- }
- }
-
- private Collection<HBaseSensitivityEntity> load(JobDataMap jobDataMap) throws Exception {
- String eagleServiceHost = (String)jobDataMap.get(EagleConfigConstants.HOST);
- Integer eagleServicePort = Integer.parseInt(jobDataMap.get(EagleConfigConstants.PORT).toString());
- String username = jobDataMap.containsKey(EagleConfigConstants.USERNAME) ? (String)jobDataMap.get(EagleConfigConstants.USERNAME) : null;
- String password = jobDataMap.containsKey(EagleConfigConstants.PASSWORD) ? (String)jobDataMap.get(EagleConfigConstants.PASSWORD) : null;
-
- // load from eagle database
- LOG.info("Load hbase resource sensitivity information from eagle service "
- + eagleServiceHost + ":" + eagleServicePort);
-
- IMetadataServiceClient client = new MetadataServiceClientImpl(eagleServiceHost, eagleServicePort, "/rest");
- return client.listHBaseSensitivities();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
index 5bdba3d..3d452b4 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
@@ -76,11 +76,11 @@ public abstract class AbstractHdfsAuditLogApplication extends StormApplication {
boltDeclarer.customGrouping("ingest", new CustomPartitionGrouping(createStrategy(config)));
}
- FileSensitivityDataJoinBolt sensitivityDataJoinBolt = new FileSensitivityDataJoinBolt(config);
+ HdfsSensitivityDataEnrichBolt sensitivityDataJoinBolt = new HdfsSensitivityDataEnrichBolt(config);
BoltDeclarer sensitivityDataJoinBoltDeclarer = builder.setBolt("sensitivityJoin", sensitivityDataJoinBolt, numOfSensitivityJoinTasks);
sensitivityDataJoinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
- IPZoneDataJoinBolt ipZoneDataJoinBolt = new IPZoneDataJoinBolt(config);
+ IPZoneDataEnrichBolt ipZoneDataJoinBolt = new IPZoneDataEnrichBolt(config);
BoltDeclarer ipZoneDataJoinBoltDeclarer = builder.setBolt("ipZoneJoin", ipZoneDataJoinBolt, numOfIPZoneJoinTasks);
ipZoneDataJoinBoltDeclarer.fieldsGrouping("sensitivityJoin", new Fields("user"));
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
deleted file mode 100644
index f2582f1..0000000
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.auditlog;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import com.typesafe.config.Config;
-import org.apache.eagle.security.auditlog.timer.FileSensitivityPollingJob;
-import org.apache.eagle.security.auditlog.util.SimplifyPath;
-import org.apache.eagle.security.entity.FileSensitivityAPIEntity;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.apache.eagle.security.util.ExternalDataJoiner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class FileSensitivityDataJoinBolt extends BaseRichBolt {
- private static Logger LOG = LoggerFactory.getLogger(FileSensitivityDataJoinBolt.class);
- private Config config;
- private OutputCollector collector;
-
- public FileSensitivityDataJoinBolt(Config config){
- this.config = config;
- }
-
-
- @Override
- public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- // start hdfs sensitivity data polling
- try{
- ExternalDataJoiner joiner = new ExternalDataJoiner(
- FileSensitivityPollingJob.class, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
- joiner.start();
- }catch(Exception ex){
- LOG.error("Fail bringing up quartz scheduler", ex);
- throw new IllegalStateException(ex);
- }
- }
-
- @Override
- public void execute(Tuple input) {
- try {
- Map<String, Object> toBeCopied = (Map<String, Object>) input.getValue(0);
- Map<String, Object> event = new TreeMap<String, Object>(toBeCopied);
- Map<String, FileSensitivityAPIEntity> map =
- (Map<String, FileSensitivityAPIEntity>) ExternalDataCache.getInstance().
- getJobResult(FileSensitivityPollingJob.class);
- FileSensitivityAPIEntity e = null;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Receive map: " + map + "event: " + event);
- }
-
- String src = (String) event.get("src");
- if (map != null && src != null) {
- String simplifiedPath = new SimplifyPath().build(src);
- for (String fileDir : map.keySet()) {
- Pattern pattern = Pattern.compile(simplifiedPath, Pattern.CASE_INSENSITIVE);
- Matcher matcher = pattern.matcher(fileDir);
- boolean isMatched = matcher.matches();
- if (isMatched) {
- e = map.get(fileDir);
- break;
- }
- }
- }
- event.put("sensitivityType", e == null ? "NA" : e.getSensitivityType());
- if (LOG.isDebugEnabled()) {
- LOG.debug("After file sensitivity lookup: " + event);
- }
- // LOG.info(">>>> After file sensitivity lookup: " + event);
- collector.emit(Arrays.asList(event.get("user"), event));
- }catch(Exception ex){
- LOG.error("error joining data, ignore it", ex);
- }finally {
- collector.ack(input);
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("user", "message"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
new file mode 100644
index 0000000..2031108
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.auditlog;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.eagle.security.auditlog.util.SimplifyPath;
+import org.apache.eagle.security.service.HdfsSensitivityEntity;
+import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HdfsSensitivityDataEnrichBolt extends AbstractDataEnrichBolt<HdfsSensitivityEntity, String> {
+ private static Logger LOG = LoggerFactory.getLogger(HdfsSensitivityDataEnrichBolt.class);
+
+ public HdfsSensitivityDataEnrichBolt(Config config){
+ super(config, new HdfsSensitivityDataEnrichLCM(config));
+ }
+
+ @Override
+ public void executeWithEnrich(Tuple input, Map<String, HdfsSensitivityEntity> map) {
+ try {
+ Map<String, Object> toBeCopied = (Map<String, Object>) input.getValue(0);
+ Map<String, Object> event = new TreeMap<String, Object>(toBeCopied);
+ HdfsSensitivityEntity e = null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Receive map: " + map + "event: " + event);
+ }
+
+ String src = (String) event.get("src");
+ if (map != null && src != null) {
+ String simplifiedPath = new SimplifyPath().build(src);
+ for (String fileDir : map.keySet()) {
+ Pattern pattern = Pattern.compile(simplifiedPath, Pattern.CASE_INSENSITIVE);
+ Matcher matcher = pattern.matcher(fileDir);
+ boolean isMatched = matcher.matches();
+ if (isMatched) {
+ e = map.get(fileDir);
+ break;
+ }
+ }
+ }
+ event.put("sensitivityType", e == null ? "NA" : e.getSensitivityType());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("After file sensitivity lookup: " + event);
+ }
+ // LOG.info(">>>> After file sensitivity lookup: " + event);
+ collector.emit(Arrays.asList(event.get("user"), event));
+ }catch(Exception ex){
+ LOG.error("error joining data, ignore it", ex);
+ }finally {
+ collector.ack(input);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("user", "message"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
new file mode 100644
index 0000000..8d2f22b
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.auditlog;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.HdfsSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.enrich.AbstractDataEnrichLCM;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public class HdfsSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HdfsSensitivityEntity, String> {
+ public HdfsSensitivityDataEnrichLCM(Config config){
+ super(config);
+ }
+ @Override
+ protected Collection<HdfsSensitivityEntity> loadFromService(ISecurityDataEnrichServiceClient client) {
+ return client.listHdfsSensitivities();
+ }
+
+ @Override
+ public String getCacheKey(HdfsSensitivityEntity entity) {
+ return entity.getFiledir();
+ }
+}