You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by mw...@apache.org on 2016/07/25 09:37:09 UTC
[32/47] incubator-eagle git commit: EAGLE-307 Update
"logstash-kafka-conf.md" Update "logstash-kafka-conf.md"
EAGLE-307 Update "logstash-kafka-conf.md"
Update "logstash-kafka-conf.md"
Author: @DadanielZ <DadanielZ>
Reviewer: @yonzhang <yo...@gmail.com>
Closes: #234
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4627eb0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4627eb0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4627eb0b
Branch: refs/heads/master
Commit: 4627eb0b576dec25c8c97d57410a044dcbd6af05
Parents: 36422ea
Author: yonzhang <yo...@gmail.com>
Authored: Thu Jun 16 15:06:36 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Thu Jun 16 15:06:36 2016 -0700
----------------------------------------------------------------------
.../src/main/docs/logstash-kafka-conf.md | 19 ++++-
.../src/main/resources/maprFSAuditLog-init.sh | 2 +-
.../security/hdfs/MAPRFSResourceConstants.java | 24 +++++++
.../hdfs/rest/MAPRFSResourceWebResource.java | 73 ++++++++++++++++++++
4 files changed, 115 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/docs/logstash-kafka-conf.md b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
index 9003fb4..342302b 100644
--- a/eagle-assembly/src/main/docs/logstash-kafka-conf.md
+++ b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
@@ -120,7 +120,7 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
type => "hdp-nn-audit"
path => "/path/to/audit.log"
start_position => end
- sincedb_path => "/var/log/logstash/"
+ sincedb_path => "/opt/logstash/sincedb.txt"
}
}
@@ -163,7 +163,7 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
type => "hdp-nn-audit"
path => "/path/to/audit.log"
start_position => end
- sincedb_path => "/var/log/logstash/"
+ sincedb_path => "/opt/logstash/sincedb.txt"
}
}
@@ -196,6 +196,21 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
}
}
+
+Notice:
+ `path => "/path/to/audit.log"` :
+ Here `path` can be configured to be a single string or an array. For example:
+ `path => [ "var/msg/example.out", /var/message/*.out", "/var/log/*.log"]`
+ `path => "/var/log/hadoop/hdfs/hdfs-audit.log.*"`
+As you can see, we can apply regex pattern to match mutiple files.
+
+
+`sincedb_path => "/opt/logstash/sincedb.txt"`:
+`sincedb_path` must be a file writable by logstash. This file will be used by Logstash to keep track of the current position in each file.
+
+
+
+
#### grok pattern testing
We have 3 typical patterns for ugi field as follows
2015-02-11 15:00:00,000 INFO FSNamesystem.audit: allowed=true ugi=user1@xyz.com (auth:TOKEN) ip=/10.115.44.55 cmd=open src=/apps/hdmi-technology/b_pulsar_coe/schema/avroschema/Session.avsc dst=null perm=null
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
index 5f99190..19eed8d 100644
--- a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
+++ b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
@@ -48,7 +48,7 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
"alias":"MapRFSAuditLogMonitor",
"groupName":"MapR",
"features":["common","metadata", "classification"],
- "config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"hdfsResource\"\n\t}\n}"
+ "config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"maprfsResource\"\n\t}\n}"
}
]
'
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
new file mode 100644
index 0000000..726ea8b
--- /dev/null
+++ b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.hdfs;
+
+public class MAPRFSResourceConstants {
+
+ public static final String MAPRFS_RESOURCE="/maprfsResource";
+ public static final String MAPRFS_APPLICATION="MAPRFSAuditLog";
+ public static final String MAPRFS_RESOURCE_RESOLVE_FORMAT_HINT ="MAPRFS Resource must be start with /";
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
new file mode 100644
index 0000000..11206bb
--- /dev/null
+++ b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.hdfs.rest;
+
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.entity.FileStatusEntity;
+import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
+import org.apache.eagle.service.common.EagleExceptionWrapper;
+import org.apache.eagle.service.security.hdfs.HDFSFileSystem;
+import org.apache.eagle.service.security.hdfs.HDFSResourceSensitivityDataJoiner;
+import org.apache.eagle.service.security.hdfs.MAPRFSResourceConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * REST Web Service to browse files and Paths in MAPRFS
+ */
+@Path(MAPRFSResourceConstants.MAPRFS_RESOURCE)
+public class MAPRFSResourceWebResource
+{
+ private static Logger LOG = LoggerFactory.getLogger(MAPRFSResourceWebResource.class);
+
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public HDFSResourceWebResponse getHDFSResource(@QueryParam("site") String site , @QueryParam("path") String filePath )
+ {
+ LOG.info("Starting MAPRFS Resource Browsing. Query Parameters ==> Site :"+site+" Path : "+filePath );
+ HDFSResourceWebResponse response = new HDFSResourceWebResponse();
+ HDFSResourceWebRequestValidator validator = new HDFSResourceWebRequestValidator();
+ MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
+ List<FileStatusEntity> result = new ArrayList<>();
+ List<FileStatus> fileStatuses = null;
+ try {
+ validator.validate(site, filePath); // First Step would be validating Request
+ Config config = repo.getConfig(MAPRFSResourceConstants.MAPRFS_APPLICATION, site);
+ Configuration conf = repo.convert(config);
+ HDFSFileSystem fileSystem = new HDFSFileSystem(conf);
+ fileStatuses = fileSystem.browse(filePath);
+ // Join with File Sensitivity Info
+ HDFSResourceSensitivityDataJoiner joiner = new HDFSResourceSensitivityDataJoiner();
+ result = joiner.joinFileSensitivity(site, fileStatuses);
+ LOG.info("Successfully browsed files in MAPRFS .");
+ } catch( Exception ex ) {
+ response.setException(EagleExceptionWrapper.wrap(ex));
+ LOG.error(" Exception When browsing Files for the MAPRFS Path :"+filePath+" " , ex);
+ }
+ response.setObj(result);
+ return response;
+ }
+}