You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/05/30 06:22:03 UTC
[2/2] incubator-eagle git commit: [EAGLE-298] Oozie auditlog
integration for Oozie security monitoring
[EAGLE-298] Oozie auditlog integration for Oozie security monitoring
- Add oozie auditlog parse code and related test code.
- Add job.html and controll.js code for new view type job.
- Add oozie auditlog entity ,oozie metatdata access config code and related test code.
https://issues.apache.org/jira/browse/EAGLE-298
Author: r7raul1984 <ta...@yhd.com>
Author: yonzhang <yo...@gmail.com>
Author: JiJun Tang <ta...@yhd.com>
Closes #208 from r7raul1984/dev.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fd39e6e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fd39e6e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fd39e6e1
Branch: refs/heads/dev
Commit: fd39e6e14ed5cdca1e7b4314b2b4e5e1d1d97425
Parents: 0a76242
Author: r7raul1984 <ta...@yhd.com>
Authored: Mon May 30 14:21:46 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon May 30 14:21:46 2016 +0800
----------------------------------------------------------------------
.../src/main/bin/eagle-topology-init.sh | 24 ++++
.../security/entity/OozieResourceEntity.java | 82 +++++++++++++
.../OozieResourceSensitivityAPIEntity.java | 46 +++++++
.../entity/SecurityEntityRepository.java | 1 +
.../audit/TestMetaDataAccessConfigRepo.java | 13 +-
.../dao/HiveSensitivityMetadataDAOImpl.java | 2 -
.../hive/resolver/HiveMetadataResolver.java | 2 -
.../eagle-security-oozie-auditlog/pom.xml | 40 +++++++
.../parse/OozieAuditLogKafkaDeserializer.java | 71 +++++++++++
.../oozie/parse/OozieAuditLogObject.java | 35 ++++++
.../oozie/parse/OozieAuditLogParser.java | 110 +++++++++++++++++
.../oozie/parse/OozieAuditLogProcessorMain.java | 33 +++++
...ozieResourceSensitivityDataJoinExecutor.java | 89 ++++++++++++++
.../OozieResourceSensitivityPollingJob.java | 63 ++++++++++
.../src/main/resources/application.conf | 66 ++++++++++
.../oozie/parse/TestOozieAuditLogParser.java | 56 +++++++++
eagle-security/eagle-security-oozie-web/pom.xml | 91 ++++++++++++++
.../OozieResourceSensitivityDataJoiner.java | 47 ++++++++
.../BadOozieMetadataAccessConfigException.java | 27 +++++
.../oozie/dao/OozieMetadataAccessConfig.java | 87 ++++++++++++++
.../oozie/dao/OozieMetadataAccessConfigDAO.java | 23 ++++
.../dao/OozieMetadataAccessConfigDAOImpl.java | 29 +++++
.../security/oozie/dao/OozieMetadataDAO.java | 25 ++++
.../oozie/dao/OozieMetadataDAOImpl.java | 54 +++++++++
.../oozie/dao/OozieSensitivityMetadataDAO.java | 29 +++++
.../dao/OozieSensitivityMetadataDAOImpl.java | 81 +++++++++++++
.../res/OozieMetadataBrowseWebResource.java | 58 +++++++++
.../res/OozieMetadataBrowseWebResponse.java | 44 +++++++
.../TestOozieResourceSensitivityDataJoiner.java | 92 ++++++++++++++
.../dao/TestOozieMetadataAccessConfig.java | 43 +++++++
.../TestOozieSensitivityMetadataDAOImpl.java | 120 +++++++++++++++++++
.../src/test/resources/coordinatorJob.json | 94 +++++++++++++++
eagle-security/pom.xml | 55 +++++----
eagle-topology-assembly/pom.xml | 42 ++++---
eagle-webservice/pom.xml | 61 +++++++---
.../public/feature/classification/controller.js | 58 ++++++++-
.../classification/page/sensitivity/job.html | 92 ++++++++++++++
pom.xml | 6 +
38 files changed, 1926 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-assembly/src/main/bin/eagle-topology-init.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh
index 9d296fc..be38a0d 100755
--- a/eagle-assembly/src/main/bin/eagle-topology-init.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh
@@ -40,6 +40,8 @@ curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Conten
curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService" -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hiveQueryLog"}, "enabled": true, "config":"classification.accessType=metastoredb_jdbc\nclassification.password=hive\nclassification.user=hive\nclassification.jdbcDriverClassName=com.mysql.jdbc.Driver\nclassification.jdbcUrl=jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true"}]'
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService" -d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"oozieAuditLog"}, "enabled": true, "config" : "classification.accessType=oozie_api\nclassification.oozieUrl=http://localhost:11000/oozie\nclassification.filter=status=RUNNING\nclassification.authType=SIMPLE"}]'
+
echo ""
echo "Importing application definitions ..."
curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService" -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hdfsAuditLog"},"description":"HDFS audit log security check application","alias":"HDFS","groupName":"DAM","features":["common","classification","userProfile","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"hdfsResource\"\n\t}\n}"}]'
@@ -48,6 +50,8 @@ curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Conten
curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService" -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"hiveQueryLog"},"description":"Hive query log security check application","alias":"HIVE","groupName":"DAM","features":["common","classification","userProfile","metadata"], "config":"{\n\t\"view\": {\n\t\t\"prefix\": \"hiveResourceSensitivity\",\n\t\t\"service\": \"HiveResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"hiveResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"table\",\n\t\t\"api\": {\n\t\t\t\"database\": \"hiveResource/databases\",\n\t\t\t\"table\": \"hiveResource/tables\",\n\t\t\t\"column\": \"hiveResource/columns\"\n\t\t},\n\t\t\"mapping\": {\n\t\t\t\"database\": \"database\",\n\t\t\t\"table\": \"table\",\n\t\t\t\"column\": \"column\"\n\t\t}\n\t}\n}"}]'
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService" -d '[{"prefix":"eagleApplicationDesc","tags":{"application":"oozieAuditLog"},"description":"Oozie audit log security check application","alias":"OOZIE","groupName":"DAM","features":["common","classification","metadata"],"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"oozieResourceSensitivity\",\n\t\t\"service\": \"OozieResourceSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"oozieResource\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"job\",\n\t\t\"api\": \"oozieResource/coordinators\"\n\t}\n}"}]'
+
echo ""
echo "Importing feature definitions ..."
curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService" -d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"common"},"description":"Provide the Policy & Alert feature.","version":"v0.3.0"}]'
@@ -139,6 +143,26 @@ curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H "Conten
-d '[ { "prefix": "alertStream", "tags": { "streamName": "userActivity", "site":"sandbox", "application":"userProfile" }, "alertExecutorIdList": [ "userProfileAnomalyDetectionExecutor" ] } ]'
#####################################################################
+# Import stream metadata for OOZIE
+#####################################################################
+
+## AlertStreamService: alert streams generated from data source
+echo ""
+echo "Importing AlertStreamService for OOZIE... "
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService" -d '[{"prefix":"alertStream","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream"},"description":"alert event stream from oozie audit log"}]'
+
+## AlertExecutorService: what alert streams are consumed by alert executor
+echo ""
+echo "Importing AlertExecutorService for OOZIE... "
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService" -d '[{"prefix":"alertExecutor","tags":{"application":"oozieAuditLog","alertExecutorId":"oozieAuditLogAlertExecutor","streamName":"oozieSecurityLogEventStream"},"description":"alert executor for oozie audit log event stream"}]'
+
+## AlertStreamSchemaServiceService: schema for event from alert stream
+echo ""
+echo "Importing AlertStreamSchemaService for OOZIE... "
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" -d '[{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"errorcode"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"httpcode"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"log level such as INFO,DEBUG","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"level"}},{"prefix":"alertStreamSchema","category":"","attrType":"long","attrDescription":"milliseconds of the dat
etime","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"timestamp"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"ip"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"user"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"app"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"group"}},{"prefix":"alertStreamSchem
a","category":"","attrType":"string","attrDescription":"such as start kill suspend resume","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"operation"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"jobId"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"status"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"errormessage"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAud
itLog","streamName":"oozieSecurityLogEventStream","attrName":"sensitivityType"}},{"prefix":"alertStreamSchema","category":"","attrType":"string","attrDescription":"","attrValueResolver":"","tags":{"application":"oozieAuditLog","streamName":"oozieSecurityLogEventStream","attrName":"parameter"}}]'
+
+
+#####################################################################
# Import notification plugin configuration into Eagle Service #
#####################################################################
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceEntity.java
new file mode 100644
index 0000000..0506dd2
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceEntity.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.entity;
+
+import com.google.common.base.Objects;
+
+import java.io.Serializable;
+
+public class OozieResourceEntity implements Serializable {
+ private String jobId;
+ private String name;
+ private String sensitiveType;
+
+ public OozieResourceEntity(String jobId, String name, String sensitiveType) {
+ this.jobId = jobId;
+ this.name = name;
+ this.sensitiveType = sensitiveType;
+ }
+
+ public OozieResourceEntity(String jobId, String name) {
+ this(jobId,name,null);
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSensitiveType() {
+ return sensitiveType;
+ }
+
+ public void setSensitiveType(String sensitiveType) {
+ this.sensitiveType = sensitiveType;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final OozieResourceEntity other = (OozieResourceEntity) obj;
+ return Objects.equal(this.jobId, other.jobId)
+ && this.name.equals(other.name)
+ && this.sensitiveType.equals(other.sensitiveType);
+ }
+
+ @Override
+ public String toString() {
+ return "OozieResourceEntity{" +
+ "jobId='" + jobId + '\'' +
+ ", name='" + name + '\'' +
+ ", sensitiveType='" + sensitiveType + '\'' +
+ '}';
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceSensitivityAPIEntity.java
new file mode 100644
index 0000000..82953f5
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/OozieResourceSensitivityAPIEntity.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.entity;
+
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Table("oozieResourceSensitivity")
+@ColumnFamily("f")
+@Prefix("oozieResourceSensitivity")
+@Service("OozieResourceSensitivityService")
+@TimeSeries(false)
+@Tags({"site", "oozieResource"})
+public class OozieResourceSensitivityAPIEntity extends TaggedLogAPIEntity {
+ private static final long serialVersionUID = 2L;
+ /**
+ * sensitivityType can be multi-value attribute, and values can be separated by "|"
+ */
+ @Column("a")
+ private String sensitivityType;
+
+ public String getSensitivityType() {
+ return sensitivityType;
+ }
+
+ public void setSensitivityType(String sensitivityType) {
+ this.sensitivityType = sensitivityType;
+ valueChanged("sensitivityType");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
index a313983..5d63690 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
@@ -28,5 +28,6 @@ public class SecurityEntityRepository extends EntityRepository {
entitySet.add(IPZoneEntity.class);
entitySet.add(HdfsUserCommandPatternEntity.class);
entitySet.add(HiveResourceSensitivityAPIEntity.class);
+ entitySet.add(OozieResourceSensitivityAPIEntity.class);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
index 3306c40..93deeab 100644
--- a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
+++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
@@ -60,5 +60,16 @@ public class TestMetaDataAccessConfigRepo {
appConfig = config.getConfig(EagleConfigConstants.APP_CONFIG);
Assert.assertTrue(appConfig.getString("envContextConfig.mode").equals("cluster"));
}
+
+ String oozieConfigStr = "classification.accessType=oozie_api\nclassification.oozieUrl=http://localhost:11000/oozie\nclassification.filter=status=RUNNING\nclassification.authType=SIMPLE";
+ config = ConfigFactory.parseString(oozieConfigStr,options);
+ Config oozieConfig = null;
+ if(config.hasPath(EagleConfigConstants.CLASSIFICATION_CONFIG)) {
+ oozieConfig = config.getConfig(EagleConfigConstants.CLASSIFICATION_CONFIG);
+ Assert.assertTrue(oozieConfig.getString("accessType").equals("oozie_api"));
+ Assert.assertTrue(oozieConfig.getString("oozieUrl").equals("http://localhost:11000/oozie"));
+ Assert.assertTrue(oozieConfig.getString("filter").equals("status=RUNNING"));
+ Assert.assertTrue(oozieConfig.getString("authType").equals("SIMPLE"));
+ }
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveSensitivityMetadataDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveSensitivityMetadataDAOImpl.java b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveSensitivityMetadataDAOImpl.java
index 90ee845..96f3f27 100644
--- a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveSensitivityMetadataDAOImpl.java
+++ b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveSensitivityMetadataDAOImpl.java
@@ -17,9 +17,7 @@
package org.apache.eagle.service.security.hive.dao;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.log.entity.ListQueryAPIResponseEntity;
import org.apache.eagle.service.generic.GenericEntityServiceResource;
-import org.apache.eagle.service.generic.ListQueryResource;
import org.apache.eagle.security.entity.HiveResourceSensitivityAPIEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/resolver/HiveMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/resolver/HiveMetadataResolver.java b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/resolver/HiveMetadataResolver.java
index fad679d..29ea183 100644
--- a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/resolver/HiveMetadataResolver.java
+++ b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/resolver/HiveMetadataResolver.java
@@ -35,8 +35,6 @@ public class HiveMetadataResolver implements AttributeResolvable<GenericAttribut
private final static Logger LOG = LoggerFactory.getLogger(HiveMetadataResolver.class);
private final static String HIVE_ATTRIBUTE_RESOLVE_FORMAT_HINT =
"hive metadata resolve must be {\"site\":\"${site}\", \"query\"=\"/{db}/{table}/{column}\"}";
- private HiveSensitivityMetadataDAOImpl dao = new HiveSensitivityMetadataDAOImpl();
- private Map<String, Map<String, String>> maps = dao.getAllHiveSensitivityMap();
@Override
public List<String> resolve(GenericAttributeResolveRequest request) throws AttributeResolveException {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/pom.xml b/eagle-security/eagle-security-oozie-auditlog/pom.xml
new file mode 100644
index 0000000..b03eafc
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ /*
+ ~ * Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ * contributor license agreements. See the NOTICE file distributed with
+ ~ * this work for additional information regarding copyright ownership.
+ ~ * The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ * (the "License"); you may not use this file except in compliance with
+ ~ * the License. You may obtain a copy of the License at
+ ~ *
+ ~ * http://www.apache.org/licenses/LICENSE-2.0
+ ~ *
+ ~ * Unless required by applicable law or agreed to in writing, software
+ ~ * distributed under the License is distributed on an "AS IS" BASIS,
+ ~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ * See the License for the specific language governing permissions and
+ ~ * limitations under the License.
+ ~ */
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>eagle-security-parent</artifactId>
+ <groupId>org.apache.eagle</groupId>
+ <version>0.4.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>eagle-security-oozie-auditlog</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-security-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogKafkaDeserializer.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogKafkaDeserializer.java
new file mode 100644
index 0000000..06d6a3e
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogKafkaDeserializer.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse;
+
+
+import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+
+public class OozieAuditLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
+ private static Logger LOG = LoggerFactory.getLogger(OozieAuditLogKafkaDeserializer.class);
+ private Properties props;
+
+ public OozieAuditLogKafkaDeserializer(Properties props){
+ this.props = props;
+ }
+
+ @Override
+ public Object deserialize(byte[] arg0) {
+ String logLine = new String(arg0);
+
+ OozieAuditLogParser parser = new OozieAuditLogParser();
+ OozieAuditLogObject entity = null;
+ try{
+ entity = parser.parse(logLine);
+ }catch(Exception ex){
+ LOG.error("Failing oozie parse audit log message", ex);
+ }
+ if(entity == null){
+ LOG.warn("Event ignored as it can't be correctly parsed, the log is ", logLine);
+ return null;
+ }
+ Map<String, Object> map = new TreeMap<String, Object>();
+ map.put("timestamp", entity.timestamp);
+ map.put("level", entity.level);
+ map.put("ip", entity.ip);
+ map.put("user", entity.user);
+ map.put("group", entity.group);
+ map.put("app", entity.app);
+ map.put("jobId", entity.jobId);
+ map.put("operation", entity.operation);
+ map.put("parameter", entity.parameter);
+ map.put("status", entity.status);
+ map.put("httpcode", entity.httpcode);
+ map.put("errorcode", entity.errorcode);
+ map.put("errormessage", entity.errormessage);
+
+ return map;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogObject.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogObject.java
new file mode 100644
index 0000000..05ab77f
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogObject.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse;
+
+public class OozieAuditLogObject {
+
+ public long timestamp;
+ public String level = "";
+ public String ip = "";
+ public String user = "";
+ public String group = "";
+ public String app = "";
+ public String jobId = "";
+ public String operation = "";
+ public String parameter = "";
+ public String status = "";
+ public String httpcode = "";
+ public String errorcode = "";
+ public String errormessage = "";
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParser.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParser.java
new file mode 100644
index 0000000..56228f8
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogParser.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.eagle.common.DateTimeUtil;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class OozieAuditLogParser {
+
+ public static final String MESSAGE_SPLIT_FLAG = "( - )";
+ private static final String COMMON_REGEX = "\\s([^\\]]*\\])";
+ public static final String ALLOW_ALL_REGEX = "(.*)";
+ private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
+ private static final String WHITE_SPACE_REGEX = "\\s+";
+ private static final String LOG_LEVEL_REGEX = "(\\w+)";
+ private static final String OOZIEAUDIT_FLAG = "(\\w+:\\d+)";
+ private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
+ + WHITE_SPACE_REGEX;
+ private final static String IP = "IP";
+ private final static String USER = "USER";
+ private final static String GROUP = "GROUP";
+ private final static String APP = "APP";
+ private final static String JOBID = "JOBID";
+ private final static String OPERATION = "OPERATION";
+ private final static String PARAMETER = "PARAMETER";
+ private final static String STATUS = "STATUS";
+ private final static String HTTPCODE = "HTTPCODE";
+ private final static String ERRORCODE = "ERRORCODE";
+ private final static String ERRORMESSAGE = "ERRORMESSAGE";
+ private static final Pattern LOG_PATTERN = constructPattern();
+
+ public OozieAuditLogObject parse(String logLine) throws Exception {
+
+ OozieAuditLogObject oozieAuditLogObject = new OozieAuditLogObject();
+ Matcher matcher = LOG_PATTERN.matcher(logLine);
+ if (!matcher.matches()) {
+ return null;
+ }
+ applyValueTo(oozieAuditLogObject, matcher);
+
+ return oozieAuditLogObject;
+ }
+
+
+ private static Pattern constructPattern() {
+ List<String> patterns = new ArrayList<String>(11);
+ patterns.add(IP);
+ patterns.add(USER);
+ patterns.add(GROUP);
+ patterns.add(APP);
+ patterns.add(JOBID);
+ patterns.add(OPERATION);
+ patterns.add(PARAMETER);
+ patterns.add(STATUS);
+ patterns.add(HTTPCODE);
+ patterns.add(ERRORCODE);
+ patterns.add(ERRORMESSAGE);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append(PREFIX_REGEX + OOZIEAUDIT_FLAG);
+ sb.append(MESSAGE_SPLIT_FLAG);
+ for (int i = 0; i < patterns.size(); i++) {
+ sb.append("(");
+ sb.append(patterns.get(i) + COMMON_REGEX);
+ sb.append(")");
+ sb.append(ALLOW_ALL_REGEX);
+ }
+ String rs = StringUtils.removeEnd(sb.toString(), ALLOW_ALL_REGEX);
+ return Pattern.compile(rs);
+ }
+
+ private void applyValueTo(OozieAuditLogObject oozieAuditLogObject, Matcher matcher) throws ParseException {
+ oozieAuditLogObject.timestamp = DateTimeUtil.humanDateToMilliseconds(matcher.group(1));
+ oozieAuditLogObject.level = matcher.group(2);
+ oozieAuditLogObject.ip = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(6), "["), "]");
+ oozieAuditLogObject.user = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(9), "["), "]");
+ oozieAuditLogObject.group = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(12), "["), "]");
+ oozieAuditLogObject.app = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(15), "["), "]");
+ oozieAuditLogObject.jobId = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(18), "["), "]");
+ oozieAuditLogObject.operation = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(21), "["), "]");
+ oozieAuditLogObject.parameter = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(24), "["), "]");
+ oozieAuditLogObject.status = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(27), "["), "]");
+ oozieAuditLogObject.httpcode = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(30), "["), "]");
+ oozieAuditLogObject.errorcode = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(33), "["), "]");
+ oozieAuditLogObject.errormessage = StringUtils.removeEnd(StringUtils.removeStart(matcher.group(36), "["), "]");
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogProcessorMain.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogProcessorMain.java
new file mode 100644
index 0000000..85dfa74
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/OozieAuditLogProcessorMain.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse;
+
+import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
+import org.apache.eagle.datastream.ExecutionEnvironments;
+import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
+import org.apache.eagle.security.oozie.parse.sensitivity.OozieResourceSensitivityDataJoinExecutor;
+
+public class OozieAuditLogProcessorMain {
+ public static void main(String[] args) throws Exception {
+ StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
+ env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
+ .flatMap(new OozieResourceSensitivityDataJoinExecutor())
+ .alertWithConsumer("oozieSecurityLogEventStream", "oozieAuditLogAlertExecutor");
+ env.execute();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinExecutor.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinExecutor.java
new file mode 100644
index 0000000..51f11af
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityDataJoinExecutor.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse.sensitivity;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.datastream.Collector;
+import org.apache.eagle.datastream.JavaStormStreamExecutor2;
+import org.apache.eagle.security.entity.OozieResourceSensitivityAPIEntity;
+import org.apache.eagle.security.util.ExternalDataCache;
+import org.apache.eagle.security.util.ExternalDataJoiner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+public class OozieResourceSensitivityDataJoinExecutor extends JavaStormStreamExecutor2<String, Map> {
+ private final static Logger LOG = LoggerFactory.getLogger(OozieResourceSensitivityDataJoinExecutor.class);
+ private Config config;
+
+ @Override
+ public void prepareConfig(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public void init() {
+ // start hive resource data polling
+ try {
+ ExternalDataJoiner joiner = new ExternalDataJoiner(OozieResourceSensitivityPollingJob.class, config);
+ joiner.start();
+ } catch (Exception ex) {
+ LOG.error("Fail to bring up quartz scheduler.", ex);
+ throw new IllegalStateException(ex);
+ }
+ }
+
+ @Override
+ public void flatMap(List<Object> input, Collector<Tuple2<String, Map>> outputCollector) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> event = (Map<String, Object>) input.get(0);
+ @SuppressWarnings("unchecked")
+ Map<String, OozieResourceSensitivityAPIEntity> map =
+ (Map<String, OozieResourceSensitivityAPIEntity>) ExternalDataCache
+ .getInstance()
+ .getJobResult(OozieResourceSensitivityPollingJob.class);
+ LOG.info(">>>> event: " + event + " >>>> map: " + map);
+
+ String resource = (String) event.get("jobId");
+
+ OozieResourceSensitivityAPIEntity sensitivityEntity = null;
+
+ if (map != null && resource != "") {
+ for (String key : map.keySet()) {
+ Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
+ if (pattern.matcher(resource).find()) {
+ sensitivityEntity = map.get(key);
+ break;
+ }
+ }
+ }
+ Map<String, Object> newEvent = new TreeMap<String, Object>(event);
+ newEvent.put("sensitivityType", sensitivityEntity == null ? "NA" : sensitivityEntity.getSensitivityType());
+ //newEvent.put("jobId", resource);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("After oozie resource sensitivity lookup: " + newEvent);
+ }
+ LOG.info("After oozie resource sensitivity lookup: " + newEvent);
+ outputCollector.collect(new Tuple2(newEvent.get("user"), newEvent));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityPollingJob.java b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityPollingJob.java
new file mode 100644
index 0000000..3ef2934
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/java/org/apache/eagle/security/oozie/parse/sensitivity/OozieResourceSensitivityPollingJob.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse.sensitivity;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import org.apache.eagle.security.entity.OozieResourceSensitivityAPIEntity;
+import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
+import org.apache.eagle.security.util.ExternalDataCache;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class OozieResourceSensitivityPollingJob extends AbstractResourceSensitivityPollingJob implements Job {
+ private final static Logger LOG = LoggerFactory.getLogger(OozieResourceSensitivityPollingJob.class);
+
+ @Override
+ public void execute(JobExecutionContext context)
+ throws JobExecutionException {
+ JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
+ try {
+ List<OozieResourceSensitivityAPIEntity>
+ oozieResourceSensitivity = load(jobDataMap, "OozieResourceSensitivityService");
+ if (oozieResourceSensitivity == null) {
+ LOG.warn("Oozie resource sensitivity information is empty");
+ return;
+ }
+ Map<String, OozieResourceSensitivityAPIEntity> map = Maps.uniqueIndex(
+ oozieResourceSensitivity,
+ new Function<OozieResourceSensitivityAPIEntity, String>() {
+ @Override
+ public String apply(OozieResourceSensitivityAPIEntity input) {
+ return input.getTags().get("oozieResource");
+ }
+ });
+ ExternalDataCache.getInstance().setJobResult(getClass(), map);
+ } catch (Exception ex) {
+ LOG.error("Fail to load oozie resource sensitivity data", ex);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/application.conf b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/application.conf
new file mode 100644
index 0000000..c9208c3
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/application.conf
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ "envContextConfig" : {
+ "env" : "storm",
+ "mode" : "local",
+ "topologyName" : "sandbox-oozieAuditLog-topology",
+ "stormConfigFile" : "security-auditlog-storm.yaml",
+ "parallelismConfig" : {
+ "kafkaMsgConsumer" : 1,
+ "oozieAuditLogAlertExecutor*" : 1
+ }
+ },
+ "dataSourceConfig": {
+ "topic" : "sandbox_oozie_audit_log",
+ "zkConnection" : "sandbox.hortonworks.com:2181",
+ "zkConnectionTimeoutMS" : 15000,
+ "consumerGroupId" : "EagleConsumer",
+ "fetchSize" : 1048586,
+ "deserializerClass" : "org.apache.eagle.security.oozie.parse.OozieAuditLogKafkaDeserializer",
+ "transactionZKServers" : "sandbox.hortonworks.com",
+ "transactionZKPort" : 2181,
+ "transactionZKRoot" : "/consumers",
+ "consumerGroupId" : "eagle.ooziesecurity.consumer",
+ "transactionStateUpdateMS" : 2000
+ },
+ "alertExecutorConfigs" : {
+ "hbaseSecurityLogAlertExecutor" : {
+ "parallelism" : 1,
+ "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
+ "needValidation" : "true"
+ }
+ },
+ "eagleProps" : {
+ "site" : "sandbox",
+ "application": "oozieAuditLog",
+ "dataJoinPollIntervalSec" : 30,
+ "mailHost" : "mailHost.com",
+ "mailSmtpPort":"25",
+ "mailDebug" : "true",
+ "eagleService": {
+ "host": "localhost",
+ "port": 9098
+ "username": "admin",
+ "password": "secret"
+ }
+ },
+ "dynamicConfigSource" : {
+ "enabled" : true,
+ "initDelayMillis" : 0,
+ "delayMillis" : 30000
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-auditlog/src/test/java/org/apache/eagle/security/oozie/parse/TestOozieAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/test/java/org/apache/eagle/security/oozie/parse/TestOozieAuditLogParser.java b/eagle-security/eagle-security-oozie-auditlog/src/test/java/org/apache/eagle/security/oozie/parse/TestOozieAuditLogParser.java
new file mode 100644
index 0000000..b2546cb
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-auditlog/src/test/java/org/apache/eagle/security/oozie/parse/TestOozieAuditLogParser.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.oozie.parse;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestOozieAuditLogParser {
+ OozieAuditLogParser parser = new OozieAuditLogParser();
+
+ @Test
+ public void testParser() throws Exception {
+
+
+ String logline = "2016-04-27 15:01:14,526 INFO oozieaudit:520 - IP [192.168.7.199], USER [tangjijun], GROUP [pms], APP [My_Workflow], JOBID [0000000-160427140648764-oozie-oozi-W], " +
+ "OPERATION [start], PARAMETER [0000000-160427140648764-oozie-oozi-W], STATUS [SUCCESS], HTTPCODE [200], ERRORCODE [501], ERRORMESSAGE [no problem]";
+ OozieAuditLogObject obj = parser.parse(logline);
+ Assert.assertEquals("192.168.7.199",obj.ip);
+ Assert.assertEquals("tangjijun",obj.user);
+ Assert.assertEquals("pms",obj.group);
+ Assert.assertEquals("My_Workflow",obj.app);
+ Assert.assertEquals("0000000-160427140648764-oozie-oozi-W",obj.jobId);
+ Assert.assertEquals("start",obj.operation);
+ Assert.assertEquals("0000000-160427140648764-oozie-oozi-W",obj.parameter);
+ Assert.assertEquals("SUCCESS",obj.status);
+ Assert.assertEquals("200",obj.httpcode);
+ Assert.assertEquals("501",obj.errorcode);
+ Assert.assertEquals("no problem",obj.errormessage);
+ Assert.assertEquals("INFO",obj.level);
+ Assert.assertEquals(1461769274526L,obj.timestamp);
+
+ }
+ @Test
+ public void testParserNotMatch() throws Exception {
+ String logline =" 2016-04-27 16:11:37,156 INFO oozieaudit:520 - Proxy user [hue] DoAs user [tangjijun] Request [http://zhangqihui:11000/oozie/v1/job/0000001-160427140648764-oozie-oozi-W?action=rerun&timezone=America%2FLos_Angeles&user.name=hue&doAs=tangjijun]";
+ OozieAuditLogObject obj = parser.parse(logline);
+ Assert.assertTrue(obj == null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/pom.xml b/eagle-security/eagle-security-oozie-web/pom.xml
new file mode 100644
index 0000000..ac11b7c
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-security-parent</artifactId>
+ <version>0.4.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>eagle-security-oozie-web</artifactId>
+ <name>eagle-security-oozie-web</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <powermock.version>1.6.5</powermock.version>
+ <easymock.version>3.4</easymock.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-entity-base</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-service-base</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.eagle</groupId>
+ <artifactId>eagle-security-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-client</artifactId>
+ <version>4.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>${easymock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/OozieResourceSensitivityDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/OozieResourceSensitivityDataJoiner.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/OozieResourceSensitivityDataJoiner.java
new file mode 100644
index 0000000..4bfe814
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/OozieResourceSensitivityDataJoiner.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie;
+
+import org.apache.eagle.security.entity.OozieResourceEntity;
+import org.apache.eagle.service.security.oozie.dao.OozieSensitivityMetadataDAO;
+import org.apache.eagle.service.security.oozie.dao.OozieSensitivityMetadataDAOImpl;
+import org.apache.oozie.client.CoordinatorJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class OozieResourceSensitivityDataJoiner {
+
+ private final static Logger LOG = LoggerFactory.getLogger(OozieResourceSensitivityDataJoiner.class);
+
+ public List<OozieResourceEntity> joinOozieResourceSensitivity(String site, List<CoordinatorJob> coordinatorJobs) {
+
+ OozieSensitivityMetadataDAO oozieSensitivityMetadataDAO = new OozieSensitivityMetadataDAOImpl();
+ Map<String, String> sensitivityMap = oozieSensitivityMetadataDAO.getOozieSensitivityMap(site);
+ LOG.info("Joining Resource with Sensitivity data ..");
+ List<OozieResourceEntity> result = new ArrayList<>();
+ for (CoordinatorJob eachJob : coordinatorJobs) {
+ OozieResourceEntity entity = new OozieResourceEntity(eachJob.getId(), eachJob.getAppName(), sensitivityMap.get(eachJob.getId()));
+ result.add(entity);
+ }
+ return result;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/BadOozieMetadataAccessConfigException.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/BadOozieMetadataAccessConfigException.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/BadOozieMetadataAccessConfigException.java
new file mode 100644
index 0000000..7c8b650
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/BadOozieMetadataAccessConfigException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+public class BadOozieMetadataAccessConfigException extends RuntimeException{
+ public BadOozieMetadataAccessConfigException(Exception ex){
+ super(ex);
+ }
+
+ public BadOozieMetadataAccessConfigException(String msg){
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfig.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfig.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfig.java
new file mode 100644
index 0000000..d2e6a76
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfig.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import com.typesafe.config.Config;
+
+public class OozieMetadataAccessConfig {
+ private String accessType;
+ private String oozieUrl;
+ private String filter;
+ private String authType;
+
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public void setAccessType(String accessType) {
+ this.accessType = accessType;
+ }
+
+ public String getFilter() {
+ return filter;
+ }
+
+ public void setFilter(String filter) {
+ this.filter = filter;
+ }
+
+ public String getAuthType() {
+ return authType;
+ }
+
+ public void setAuthType(String authType) {
+ this.authType = authType;
+ }
+
+ public String getOozieUrl() {
+ return oozieUrl;
+ }
+
+ public void setOozieUrl(String oozieUrl) {
+ this.oozieUrl = oozieUrl;
+ }
+
+ @Override
+ public String toString() {
+ return "accessType:" + accessType + ",oozieUrl:" + oozieUrl + ",filter:" + filter + ",authType:" + authType;
+ }
+
+ public final static class OOZIECONF {
+ public final static String ACCESSTYPE = "accessType";
+ public final static String OOZIEURL = "oozieUrl";
+ public final static String FILTER = "filter";
+ public final static String AUTHTYPE = "authType";
+ }
+
+ public static OozieMetadataAccessConfig config2Entity(Config config) {
+ OozieMetadataAccessConfig oozieconf = new OozieMetadataAccessConfig();
+ if(config.hasPath(OOZIECONF.ACCESSTYPE)) {
+ oozieconf.setAccessType(config.getString(OOZIECONF.ACCESSTYPE));
+ }
+ if(config.hasPath(OOZIECONF.OOZIEURL)) {
+ oozieconf.setOozieUrl(config.getString(OOZIECONF.OOZIEURL));
+ }
+ if(config.hasPath(OOZIECONF.FILTER)) {
+ oozieconf.setFilter(config.getString(OOZIECONF.FILTER));
+ }
+ if(config.hasPath(OOZIECONF.AUTHTYPE)) {
+ oozieconf.setAuthType(config.getString(OOZIECONF.AUTHTYPE));
+ }
+ return oozieconf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAO.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAO.java
new file mode 100644
index 0000000..7a253fb
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAO.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+public interface OozieMetadataAccessConfigDAO {
+
+ OozieMetadataAccessConfig getConfig(String site) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAOImpl.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAOImpl.java
new file mode 100644
index 0000000..8cab617
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataAccessConfigDAOImpl.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
+
+public class OozieMetadataAccessConfigDAOImpl implements OozieMetadataAccessConfigDAO{
+ @Override
+ public OozieMetadataAccessConfig getConfig(String site) throws Exception{
+ MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
+ Config config = repo.getConfig("oozieAuditLog", site);
+ return OozieMetadataAccessConfig.config2Entity(config);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAO.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAO.java
new file mode 100644
index 0000000..c8461a9
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAO.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import org.apache.oozie.client.CoordinatorJob;
+
+import java.util.List;
+
+public interface OozieMetadataDAO {
+ List<CoordinatorJob> getCoordJobs() throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAOImpl.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAOImpl.java
new file mode 100644
index 0000000..65b4bcb
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieMetadataDAOImpl.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import org.apache.oozie.client.AuthOozieClient;
+import org.apache.oozie.client.CoordinatorJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+public class OozieMetadataDAOImpl implements OozieMetadataDAO {
+
+
+ private static Logger LOG = LoggerFactory.getLogger(OozieMetadataDAOImpl.class);
+
+ private OozieMetadataAccessConfig config;
+
+ public OozieMetadataDAOImpl(OozieMetadataAccessConfig config) {
+ if (config.getAccessType() == null)
+ throw new BadOozieMetadataAccessConfigException("access Type is null, options: [oozie_api]");
+ this.config = config;
+ }
+
+ private AuthOozieClient createOozieConnection() throws Exception {
+ AuthOozieClient client = new AuthOozieClient(config.getOozieUrl(), config.getAuthType());
+ return client;
+ }
+
+ @Override
+ public List<CoordinatorJob> getCoordJobs() throws Exception {
+ AuthOozieClient client = createOozieConnection();
+ List<CoordinatorJob> jobs = client.getCoordJobsInfo(config.getFilter(), 0, Integer.MAX_VALUE);
+ if (jobs == null || jobs.isEmpty()) {
+ return Collections.emptyList();
+ }
+ return jobs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAO.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAO.java
new file mode 100644
index 0000000..dc16d01
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAO.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import java.util.Map;
+
+/**
+ * retrieve oozie sensitivity metadata directly from table
+ */
+public interface OozieSensitivityMetadataDAO {
+ // site to (resource, sensitivityType)
+ Map<String, Map<String, String>> getAllOozieSensitivityMap();
+ // resource to sensitivityType
+ Map<String, String> getOozieSensitivityMap(String site);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAOImpl.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAOImpl.java
new file mode 100644
index 0000000..247eb90
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/dao/OozieSensitivityMetadataDAOImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.dao;
+
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.security.entity.OozieResourceSensitivityAPIEntity;
+import org.apache.eagle.service.generic.GenericEntityServiceResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class OozieSensitivityMetadataDAOImpl implements OozieSensitivityMetadataDAO{
+ private static Logger LOG = LoggerFactory.getLogger(OozieSensitivityMetadataDAOImpl.class);
+
+ @Override
+ public Map<String, Map<String, String>> getAllOozieSensitivityMap(){
+ GenericEntityServiceResource resource = new GenericEntityServiceResource();
+ /* parameters are: query, startTime, endTime, pageSzie, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing,
+ * parallel, metricName*/
+ GenericServiceAPIResponseEntity ret = resource.search("OozieResourceSensitivityService[]{*}", null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false,
+ 0, null, false);
+ List<OozieResourceSensitivityAPIEntity> list = (List<OozieResourceSensitivityAPIEntity>) ret.getObj();
+ if( list == null )
+ return Collections.emptyMap();
+ Map<String, Map<String, String>> res = new HashMap<String, Map<String, String>>();
+
+ for(OozieResourceSensitivityAPIEntity entity : list){
+ String site = entity.getTags().get("site");
+ if(entity.getTags().containsKey("oozieResource")) {
+ if(res.get(site) == null){
+ res.put(site, new HashMap<String, String>());
+ }
+ Map<String, String> resSensitivityMap = res.get(site);
+ resSensitivityMap.put(entity.getTags().get("oozieResource"), entity.getSensitivityType());
+ }
+ else {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("An invalid sensitivity entity is detected" + entity);
+ }
+ }
+ }
+ return res;
+ }
+
+ @Override
+ public Map<String, String> getOozieSensitivityMap(String site){
+ GenericEntityServiceResource resource = new GenericEntityServiceResource();
+ String queryFormat = "OozieResourceSensitivityService[@site=\"%s\"]{*}";
+ GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat, site), null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false,
+ 0, null, false);
+ List<OozieResourceSensitivityAPIEntity> list = (List<OozieResourceSensitivityAPIEntity>) ret.getObj();
+ if( list == null )
+ return Collections.emptyMap();
+ Map<String, String> resSensitivityMap = new HashMap<String, String>();
+ for(OozieResourceSensitivityAPIEntity entity : list){
+ if(entity.getTags().containsKey("oozieResource")) {
+ resSensitivityMap.put(entity.getTags().get("oozieResource"), entity.getSensitivityType());
+ }
+ }
+ return resSensitivityMap;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResource.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResource.java
new file mode 100644
index 0000000..e0fe57f
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResource.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.res;
+
+import org.apache.eagle.security.entity.OozieResourceEntity;
+import org.apache.eagle.service.common.EagleExceptionWrapper;
+import org.apache.eagle.service.security.oozie.OozieResourceSensitivityDataJoiner;
+import org.apache.eagle.service.security.oozie.dao.*;
+import org.apache.oozie.client.CoordinatorJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.util.*;
+
+@Path("/oozieResource")
+public class OozieMetadataBrowseWebResource {
+
+ private static Logger LOG = LoggerFactory.getLogger(OozieMetadataBrowseWebResource.class);
+
+ @Path("/coordinators")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public OozieMetadataBrowseWebResponse getCoordJobs(@QueryParam("site") String site){
+
+ OozieMetadataBrowseWebResponse response = new OozieMetadataBrowseWebResponse();
+ List<CoordinatorJob> coordinators = null;
+ try {
+ OozieMetadataAccessConfig config = new OozieMetadataAccessConfigDAOImpl().getConfig(site);
+ OozieMetadataDAO dao = new OozieMetadataDAOImpl(config);
+ coordinators = dao.getCoordJobs();
+ } catch(Exception ex){
+ LOG.error("fail getting coordinators", ex);
+ response.setException(EagleExceptionWrapper.wrap(ex));
+ }
+ List<OozieResourceEntity> result = new ArrayList<>();
+ OozieResourceSensitivityDataJoiner joiner = new OozieResourceSensitivityDataJoiner();
+ result = joiner.joinOozieResourceSensitivity(site, coordinators);
+ response.setObj(result);
+ return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fd39e6e1/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResponse.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResponse.java b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResponse.java
new file mode 100644
index 0000000..9f5f4ce
--- /dev/null
+++ b/eagle-security/eagle-security-oozie-web/src/main/java/org/apache/eagle/service/security/oozie/res/OozieMetadataBrowseWebResponse.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.oozie.res;
+
+import org.apache.eagle.security.entity.OozieResourceEntity;
+
+import java.util.List;
+
+public class OozieMetadataBrowseWebResponse {
+ private String exception;
+
+ public List<OozieResourceEntity> getObj() {
+ return obj;
+ }
+
+ public void setObj(List<OozieResourceEntity> obj) {
+ this.obj = obj;
+ }
+
+ private List<OozieResourceEntity> obj;
+
+ public String getException() {
+ return exception;
+ }
+
+ public void setException(String exception) {
+ this.exception = exception;
+ }
+
+}