You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/06/10 01:29:26 UTC
[1/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Repository: incubator-eagle
Updated Branches:
refs/heads/develop eef4930c3 -> 8b646715e
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelerTest.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelerTest.java b/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelerTest.java
deleted file mode 100644
index 89f7386..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelerTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.security.userprofile.model.kde.UserProfileKDEModeler;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.junit.Assert;
-
-public class UserProfileKDEModelerTest {
-
- @org.junit.Test
- public void testBuild() throws Exception {
- UserProfileKDEModeler modeler = new UserProfileKDEModeler();
- String user = "user1";
- final RealMatrix mockMatrix = new Array2DRowRealMatrix(buildMockData());
- scala.collection.immutable.List<UserProfileKDEModel> testStatistics = modeler.build("default",user, mockMatrix);
- Assert.assertEquals(testStatistics.length(), 1);
- UserProfileKDEModel model = testStatistics.head();
- Assert.assertEquals(model.user(), "user1");
- Assert.assertEquals(model.statistics().length, mockMatrix.getColumnDimension());
- Assert.assertEquals(-36, model.minProbabilityEstimate(), 1.0);
- Assert.assertEquals(-37, model.maxProbabilityEstimate(), 1.0);
- Assert.assertEquals(-37, model.nintyFivePercentileEstimate(), 1.0);
- Assert.assertEquals(-37, model.medianProbabilityEstimate(), 1.0);
- Assert.assertEquals(true, model.statistics()[3].isLowVariant());
- Assert.assertEquals(false, model.statistics()[0].isLowVariant());
- Assert.assertEquals("setTimes", model.statistics()[3].getCommandName());
- Assert.assertEquals("getfileinfo", model.statistics()[0].getCommandName());
- }
-
- private double[][] buildMockData(){
- double[][] mockData = new double[][]{
- {114503.0,2.8820906E7,123618.0,0.0,64.0,15772.0,186.0,24296.0,12.0,9.0,32.0},
- {53300.0,390772.0,157626.0,0.0,67.0,10501.0,226.0,8567.0,14.0,0.0,12.0},
- {25659.0,140858.0,35731.0,0.0,169.0,1619.0,520.0,2965.0,34.0,0.0,34.0}
- };
- return mockData;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/test/scala/org/apache/eagle/security/userprofile/daemon/UtilsSpec.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/test/scala/org/apache/eagle/security/userprofile/daemon/UtilsSpec.scala b/eagle-security/eagle-security-userprofile/training/src/test/scala/org/apache/eagle/security/userprofile/daemon/UtilsSpec.scala
deleted file mode 100644
index 18ef625..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/test/scala/org/apache/eagle/security/userprofile/daemon/UtilsSpec.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import org.apache.eagle.common.DateTimeUtil
-import org.joda.time.Period
-import org.scalatest._
-
-/**
- * @since 9/2/15
- */
-class UtilsSpec extends FlatSpec with Matchers{
- import org.apache.eagle.security.userprofile.daemon.Utils._
-
- it should "formatPathWithMilliseconds " in {
- formatPathWithMilliseconds("/tmp/path/to/log.${yyyy-MM-dd-hh}.gz")(DateTimeUtil.humanDateToMilliseconds("2015-09-02 10:32:17,000")) should be ("/tmp/path/to/log.2015-09-02-10.gz")
- formatPathWithMilliseconds("/tmp/path/to/${yyyy-MM-dd}/*.tar.gz")(DateTimeUtil.humanDateToMilliseconds("2015-09-02 10:32:17,000")) should be ("/tmp/path/to/2015-09-02/*.tar.gz")
- formatPathWithMilliseconds("/tmp/path/to/${yyyy-MM-dd}/${yyyy-MM-dd-hh}.tar.gz")(DateTimeUtil.humanDateToMilliseconds("2015-09-02 10:32:17,000")) should be ("/tmp/path/to/2015-09-02/2015-09-02-10.tar.gz")
- }
-
- it should "formatPathsInDuration" in {
- val pathes:Seq[String] = formatPathsInDuration("/tmp/path/to/log.${yyyy-MM-dd-hh}.gz",DateTimeUtil.humanDateToMilliseconds("2015-09-02 10:32:17,000"),Period.parse("P30D"))
- //pathes.length should be (30 * 24)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/pom.xml b/eagle-security/pom.xml
index e1a6b6f..6b9a7af 100644
--- a/eagle-security/pom.xml
+++ b/eagle-security/pom.xml
@@ -35,7 +35,6 @@
<module>eagle-security-common</module>
<module>eagle-security-hdfs-auditlog</module>
<module>eagle-security-maprfs-auditlog</module>
- <module>eagle-security-userprofile</module>
<module>eagle-security-hive</module>
<module>eagle-security-hive-web</module>
<module>eagle-security-hdfs-web</module>
[7/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Author: @yonzhang2012 <yo...@apache.org>
Reviewer:
Closes: #233
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/8b646715
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/8b646715
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/8b646715
Branch: refs/heads/develop
Commit: 8b646715edbe8f0e3a4a2d926c10e4a942117085
Parents: eef4930
Author: yonzhang <yo...@gmail.com>
Authored: Thu Jun 9 18:31:58 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Thu Jun 9 18:31:58 2016 -0700
----------------------------------------------------------------------
.../eagle-security-hdfs-authlog/README.md | 42 +
.../eagle-security-userprofile/common/pom.xml | 127 --
.../common/run_mltraining_topology.sh | 19 -
.../security/userprofile/UserPartitioner.java | 32 -
.../userprofile/UserProfileConstants.java | 37 -
.../security/userprofile/UserProfileUtils.java | 53 -
.../model/ScheduleCommandEntity.java | 73 -
.../model/UserActivityAggModelDeserializer.java | 52 -
.../model/UserActivityAggModelEntity.java | 86 -
.../model/UserCommandStatistics.java | 83 -
.../model/UserProfileEigenModelEntity.java | 204 --
.../model/UserProfileEntityRepository.java | 25 -
.../model/UserProfileKDEModelEntity.java | 129 --
.../userprofile/model/package-info.java | 20 -
.../common/src/main/resources/log4j.properties | 35 -
.../common/src/main/resources/storm.yaml.1 | 29 -
.../userprofile/model/AuditlogParser.scala | 54 -
.../userprofile/model/EntityConversion.scala | 27 -
.../model/UserActivityAggModel.scala | 28 -
.../model/UserProfileEigenModel.scala | 44 -
.../userprofile/model/UserProfileKDEModel.scala | 37 -
.../userprofile/model/UserProfileModel.scala | 22 -
.../common/TestUserProfileUtils.java | 106 -
.../common/src/test/resources/log4j.properties | 35 -
.../common/src/test/resources/storm.yaml.2 | 29 -
.../detection/dev-supports/prepare-metadata.sh | 45 -
.../detection/dev-supports/prepare-table.sh | 21 -
.../produce-useractivity-with-spark.sh | 18 -
.../dev-supports/produce-useractivity.sh | 18 -
.../dev-supports/useractivity-agg-json.txt | 16 -
.../userprofile-executor-definition.json | 1 -
.../userprofile-policy-definition.json | 1 -
.../userprofile-stream-description.json | 1 -
.../userprofile-stream-metadata.json | 1 -
.../detection/pom.xml | 86 -
.../detection/run_onlineprediction_topology.sh | 19 -
.../eagle/security/userprofile/TimeWindow.java | 45 -
.../userprofile/UserActivityAggregator.java | 34 -
.../UserProfileAggregatorExecutor.java | 79 -
.../userprofile/UserProfileAnomalyDetector.java | 44 -
.../UserProfileDetectionBatchMain.java | 68 -
.../UserProfileDetectionConstants.java | 25 -
.../userprofile/UserProfileDetectionMain.java | 43 -
.../UserProfileDetectionStreamMain.java | 56 -
.../UserProfileMLAlgorithmEvaluator.java | 118 --
.../impl/AbstractUserProfileEigenEvaluator.java | 50 -
.../impl/AbstractUserProfileKDEEvaluator.java | 40 -
.../userprofile/impl/TimeWindowImpl.java | 87 -
.../impl/UserActivityAggregatorImpl.java | 169 --
.../impl/UserProfileAnomalyEigenEvaluator.java | 173 --
.../impl/UserProfileAnomalyKDEEvaluator.java | 142 --
....eagle.policy.PolicyEvaluatorServiceProvider | 16 -
.../src/main/resources/application-batch.conf | 63 -
.../src/main/resources/application-stream.conf | 65 -
.../src/main/resources/log4j.properties | 35 -
.../main/resources/ml-policyDef-UserProfile.txt | 33 -
.../src/main/resources/samplePolicy.json | 18 -
.../src/main/resources/userprofile.storm.yaml | 29 -
.../userprofile/TestUserActivityAggregator.java | 104 -
.../TestUserProfileAnomalyEigenEvaluator.java | 115 --
....eagle.policy.PolicyEvaluatorServiceProvider | 16 -
.../test/resources/ml-policyDef-UserProfile.txt | 1 -
.../detection/src/test/resources/storm.yaml.1 | 29 -
.../eagle-security-userprofile/pom.xml | 37 -
.../training/README.md | 54 -
.../training/dev-supports/run-scheduler.sh | 48 -
.../training/dev-supports/run-training.sh | 25 -
.../training/dev-supports/run-with-kafka.sh | 30 -
.../training/dev-supports/run-with-localfile.sh | 33 -
.../training/dev-supports/run-with-service.sh | 33 -
.../training/dev-supports/run-with-spark.sh | 52 -
.../dev-supports/trigger-onetime-training.sh | 23 -
.../eagle-security-userprofile/training/pom.xml | 219 --
.../training/src/assembly/assembly.xml | 67 -
.../model/JavaUserProfileModeler.java | 50 -
.../model/eigen/UserProfileEigenModeler.java | 297 ---
.../model/kde/UserProfileKDEModeler.java | 147 --
.../src/main/resources/application.conf | 36 -
.../training/src/main/resources/hdfs-audit.log | 1920 ------------------
.../src/main/resources/log4j.properties | 35 -
.../training/src/main/resources/reference.conf | 35 -
.../main/sbin/submit-userprofile-training.sh | 23 -
.../userprofile/UserProfileJobFactory.scala | 31 -
.../userprofile/UserProfileTrainingApp.scala | 80 -
.../userprofile/UserProfileTrainingCLI.scala | 58 -
.../security/userprofile/daemon/Command.scala | 116 --
.../security/userprofile/daemon/Consumers.scala | 83 -
.../userprofile/daemon/Coordinator.scala | 58 -
.../security/userprofile/daemon/Producers.scala | 154 --
.../userprofile/daemon/SchedulePolicy.scala | 100 -
.../security/userprofile/daemon/Scheduler.scala | 62 -
.../userprofile/daemon/SchedulerContext.scala | 143 --
.../daemon/UserProfileCommandDao.scala | 90 -
.../security/userprofile/daemon/Utils.scala | 77 -
.../job/AuditLogTrainingSparkJob.scala | 157 --
.../job/UserProfileTrainingJob.scala | 64 -
.../userprofile/model/UserProfileContext.scala | 22 -
.../userprofile/model/UserProfileModeler.scala | 36 -
.../userprofile/sink/UserActivityAggSink.scala | 84 -
.../userprofile/sink/UserProfileModelSink.scala | 89 -
.../model/UserProfileEigenModelerTest.java | 56 -
.../model/UserProfileKDEModelerTest.java | 54 -
.../security/userprofile/daemon/UtilsSpec.scala | 39 -
eagle-security/pom.xml | 1 -
104 files changed, 42 insertions(+), 8128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-hdfs-authlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-authlog/README.md b/eagle-security/eagle-security-hdfs-authlog/README.md
index d77ccf7..e1a1971 100644
--- a/eagle-security/eagle-security-hdfs-authlog/README.md
+++ b/eagle-security/eagle-security-hdfs-authlog/README.md
@@ -27,3 +27,45 @@ limitations under the License.
#### consume parsed hdfs authorization log
/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --topic sandbox_hdfs_auth_log --zookeeper sandbox.hortonworks.com:2181
+
+
+#### run logstash in sandbox to fetch Hdfs authorization log
+
+##### step 1: create logstash config file: hdfs-authlog.conf
+
+~~~
+ input {
+ file {
+ type => "hdfs-auth"
+ path => "/var/log/hadoop/hdfs/SecurityAuth.audit"
+ start_position => end
+ sincedb_path => "/var/log/logstash/sincedb"
+ }
+ }
+
+ output {
+ if [type] == "hdfs-auth" {
+ kafka {
+ codec => plain {
+ format => "%{message}"
+ }
+ bootstrap_servers => "sandbox.hortonworks.com:6667"
+ topic_id => "sandbox_hdfs_auth_log"
+ acks => "0"
+ timeout_ms => 10000
+ retries => 3
+ retry_backoff_ms => 100
+ batch_size => 16384
+ send_buffer_bytes => 131072
+ client_id => "hdfs-auth"
+ }
+ # stdout { codec => rubydebug }
+ }
+ }
+
+~~~
+
+##### step 2: run logstash
+
+bin/logstash -f hdfs-authlog.conf
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/pom.xml b/eagle-security/eagle-security-userprofile/common/pom.xml
deleted file mode 100644
index 1faee1a..0000000
--- a/eagle-security/eagle-security-userprofile/common/pom.xml
+++ /dev/null
@@ -1,127 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-userprofile-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <packaging>jar</packaging>
- <artifactId>eagle-security-userprofile-common</artifactId>
- <name>eagle-security-userprofile-common</name>
- <url>http://maven.apache.org</url>
-
- <dependencies>
- <dependency>
- <groupId>com.googlecode.efficient-java-matrix-library</groupId>
- <artifactId>ejml</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>joda-time</artifactId>
- <groupId>joda-time</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-alert-process</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-machinelearning-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <!--<sourceDirectory>.</sourceDirectory>-->
- <!--<testSourceDirectory>src/test/scala</testSourceDirectory>-->
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <useFile>false</useFile>
- <disableXmlReport>true</disableXmlReport>
- <includes>
- <include>**/*Test.*</include>
- <include>**/Test*.*</include>
- <include>**/*Suite.*</include>
- </includes>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/run_mltraining_topology.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/run_mltraining_topology.sh b/eagle-security/eagle-security-userprofile/common/run_mltraining_topology.sh
deleted file mode 100755
index dfab19d..0000000
--- a/eagle-security/eagle-security-userprofile/common/run_mltraining_topology.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.7.0_51.jdk/Contents/Home
-mvn -X exec:java -Dexec.mainClass="eagle.auditlog.userprofile.training.AuditLogUserProfileMain"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserPartitioner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserPartitioner.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserPartitioner.java
deleted file mode 100644
index 1a6023b..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserPartitioner.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import kafka.utils.VerifiableProperties;
-import kafka.utils.Utils;
-
-public class UserPartitioner implements kafka.producer.Partitioner{
- public UserPartitioner(VerifiableProperties prop){
-
- }
-
- @Override
- public int partition(Object arg0, int arg1) {
- String user = (String)arg0;
- return Utils.abs(user.hashCode()) % arg1;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileConstants.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileConstants.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileConstants.java
deleted file mode 100644
index 4700cf3..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileConstants.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.ml.MLConstants;
-
-import java.io.Serializable;
-
-public final class UserProfileConstants implements Serializable{
- public static final String[] DEFAULT_CMD_TYPES= new String[]{"getfileinfo", "open", "listStatus", "setTimes", "setPermission", "rename", "mkdirs", "create", "setReplication", "contentSummary", "delete", "setOwner", "fsck"};
- public static final String DEFAULT_TRAINING_APP_NAME = "AuditlogTraining";
-
- public static final String USER_PROFILE_EIGEN_MODEL_SERVICE = "UserProfileEigenModelService";
- public static final String USER_PROFILE_KDE_MODEL_SERVICE = "UserProfileKDEModelService";
- public static final String USER_ACTIVITY_AGG_MODEL_SERVICE = "UserActivityAggModelService";
-
- public static final String EIGEN_DECOMPOSITION_ALGORITHM = "EigenDecomposition";
- public static final String KDE_ALGORITHM = "DE";
-
- public static final String USER_TAG = "user";
- public static final String SITE_TAG = "site";
- public static final String ALGORITHM_TAG = MLConstants.ALGORITHM;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileUtils.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileUtils.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileUtils.java
deleted file mode 100644
index 00d8e28..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/UserProfileUtils.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import org.joda.time.Period;
-import org.joda.time.Seconds;
-import scala.Int;
-
-public class UserProfileUtils {
- /**
- * For example: timestamp stands for time: 1990/01/07 12:45 and period is PT30, then result is 1990/01/07 12:30
- *
- * @param seconds
- * @param period
- *
- * @return formatted timestamp
- */
- public static long formatSecondsByPeriod(long seconds,Seconds period){
- return seconds - (seconds % Int.int2long(period.getSeconds()));
- }
-
- /**
- * @param seconds
- * @param period
- * @return
- */
- public static long formatSecondsByPeriod(long seconds,Period period){
- return seconds - (seconds % Int.int2long(period.toStandardSeconds().getSeconds()));
- }
-
- /**
- * @param milliseconds
- * @param period
- * @return milliseconds
- */
- public static long formatMillisecondsByPeriod(long milliseconds,Period period){
- return formatSecondsByPeriod(milliseconds/1000,period)*1000;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/ScheduleCommandEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/ScheduleCommandEntity.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/ScheduleCommandEntity.java
deleted file mode 100644
index 9727572..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/ScheduleCommandEntity.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-
-@Service(ScheduleCommandEntity.ScheduleTaskService)
-@Table("userprofile")
-@Prefix("schedule_command")
-@ColumnFamily("f")
-@TimeSeries(true)
-@Tags({"site","type"})
-public class ScheduleCommandEntity extends TaggedLogAPIEntity {
- public static final String ScheduleTaskService = "ScheduleTaskService";
- public static enum STATUS {
- INITIALIZED, PENDING,EXECUTING, SUCCEEDED, FAILED
- // , CANCELED
- }
-
- @Column("a")
- private String status;
- public static String getScheduleTaskService() {
- return ScheduleTaskService;
- }
-
- public String getStatus() {
- return status;
- }
-
- public void setStatus(String status) {
- this.status = status;
- valueChanged("status");
- }
-
- public long getUpdateTime() {
- return updateTime;
- }
-
- public void setUpdateTime(long updateTime) {
- this.updateTime = updateTime;
- valueChanged("updateTime");
- }
-
- public String getDetail() {
- return detail;
- }
-
- public void setDetail(String detail) {
- this.detail = detail;
- valueChanged("detail");
- }
-
- @Column("b")
- private String detail;
- @Column("c")
- private long updateTime;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelDeserializer.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelDeserializer.java
deleted file mode 100644
index fc9c8ad..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelDeserializer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Properties;
-
-public class UserActivityAggModelDeserializer implements SpoutKafkaMessageDeserializer {
- private static Logger LOG = LoggerFactory.getLogger(UserActivityAggModelDeserializer.class);
- private final Properties props;
- private final static String CHARSET = "UTF-8";
-
- public UserActivityAggModelDeserializer(Properties props){
- this.props = props;
- }
-
- @Override
- public Object deserialize(byte[] bytes) {
- String str = null;
- try {
- str = new String(bytes,CHARSET);
- return TaggedLogAPIEntity.buildObjectMapper().readValue(str, UserActivityAggModelEntity.class);
- } catch (UnsupportedEncodingException e) {
- LOG.error("Not support encoding: " + CHARSET, e);
- throw new IllegalStateException("Not support "+CHARSET,e);
- } catch (IOException e) {
- LOG.warn("Failed to deserialize as UserActivityAggModel, ignored: "+str,e);
-// throw new IllegalArgumentException("Failed to deserialize: "+str,e);
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelEntity.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelEntity.java
deleted file mode 100644
index bbb0edf..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserActivityAggModelEntity.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import scala.collection.JavaConversions;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@Table("userprofile")
-@ColumnFamily("f")
-@Prefix("useractivity")
-@Service(UserProfileConstants.USER_ACTIVITY_AGG_MODEL_SERVICE)
-@TimeSeries(true)
-@JsonIgnoreProperties(ignoreUnknown = true)
-@Tags({"site","user"})
-public class UserActivityAggModelEntity extends TaggedLogAPIEntity{
- @Column("a")
- private List<String> cmdTypes;
- @Column("b")
- private double[][] cmdMatrix;
-
- public double[][] getCmdMatrix() {
- return cmdMatrix;
- }
-
- public void setCmdMatrix(double[][] cmdMatrix) {
- this.cmdMatrix = cmdMatrix;
- valueChanged("cmdMatrix");
- }
-
- public List<String> getCmdTypes() {
- return cmdTypes;
- }
-
- public void setCmdTypes(List<String> cmdTypes) {
- this.cmdTypes = cmdTypes;
- valueChanged("cmdTypes");
- }
-
- public static UserActivityAggModel toModel(final UserActivityAggModelEntity entity){
- //return null;
- UserActivityAggModel model = new UserActivityAggModel(
- entity.getTags().get(UserProfileConstants.USER_TAG),
- new Array2DRowRealMatrix(entity.getCmdMatrix()),
- JavaConversions.<String>asScalaBuffer(entity.getCmdTypes()),
- entity.getTags().get(UserProfileConstants.SITE_TAG),
- entity.getTimestamp());
- return model;
- }
-
- public static UserActivityAggModelEntity fromModel(final UserActivityAggModel model){
- UserActivityAggModelEntity entity = new UserActivityAggModelEntity();
- entity.setCmdTypes(JavaConversions.seqAsJavaList(model.cmdTypes()));
- entity.setCmdMatrix(model.matrix().getData());
- Map<String,String> tags = new HashMap<String,String>(){{
- put(UserProfileConstants.SITE_TAG,model.site());
- put(UserProfileConstants.USER_TAG,model.user());
- }};
- entity.setTimestamp(model.timestamp());
- entity.setTags(tags);
- return entity;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserCommandStatistics.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserCommandStatistics.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserCommandStatistics.java
deleted file mode 100644
index 2e5838c..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserCommandStatistics.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-public class UserCommandStatistics implements Serializable {
- private static final long serialVersionUID = -7145615521424420913L;
- private String commandName;
- private double mean;
- private double stddev;
- private boolean isLowVariant;
-
- public UserCommandStatistics(){}
-
- public UserCommandStatistics(Map<String, Object> rawMap){
- if(rawMap.get(COMMAND_NAME)!=null) this.commandName = (String) rawMap.get(COMMAND_NAME);
- if(rawMap.get(MEAN)!=null)this.mean = (double) rawMap.get(MEAN);
- if(rawMap.get(STDDEV)!=null) this.stddev = (double) rawMap.get(STDDEV);
- if(rawMap.get(IS_LOW_VARIANT)!=null) this.isLowVariant = (boolean) rawMap.get(IS_LOW_VARIANT);
- }
-
- public String getCommandName() {
- return commandName;
- }
- public void setCommandName(String commandName) {
- this.commandName = commandName;
- }
-
- public double getMean() {
- return mean;
- }
-
- public void setMean(double mean) {
- this.mean = mean;
- }
-
- public double getStddev() {
- return stddev;
- }
-
- public void setStddev(double stddev) {
- this.stddev = stddev;
- }
-
- public boolean isLowVariant() {
- return isLowVariant;
- }
-
- public void setLowVariant(boolean isLowVariant) {
- this.isLowVariant = isLowVariant;
- }
-
- public final static String COMMAND_NAME = "commandName";
- public final static String MEAN = "mean";
- public final static String STDDEV = "stddev";
- public final static String IS_LOW_VARIANT = "isLowVariant";
-
- public Map<String,Object> toMap(){
- Map<String,Object> map = new HashMap<>();
- map.put(COMMAND_NAME,this.commandName);
- map.put(MEAN,this.mean);
- map.put(STDDEV,this.stddev);
- map.put(IS_LOW_VARIANT,this.isLowVariant);
- return map;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelEntity.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelEntity.java
deleted file mode 100644
index 2651f51..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelEntity.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class UserProfileEigenModelEntity {
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileEigenModelEntity.class);
-
- public static MLModelAPIEntity serializeModel(UserProfileEigenModel model) throws IOException {
- if(model == null) return null;
- if(model.user()==null) throw new IllegalArgumentException("user is null");
-
- MLModelAPIEntity entity = new MLModelAPIEntity();
- UserProfileEigenModelEntity content = new UserProfileEigenModelEntity();
- entity.setVersion(model.version());
- if(model.uMatrix()!=null) content.setUMatrix(model.uMatrix().getData());
- if(model.diagonalMatrix()!=null) content.setDiagonalMatrix(model.diagonalMatrix().getData());
- content.setDimension(model.dimension());
- if(model.minVector()!=null) content.setMinVector(model.minVector().toArray());
- if(model.maxVector()!=null) content.setMaxVector(model.maxVector().toArray());
-
- RealVector[] pcsInVector = model.principalComponents();
- if(pcsInVector!=null) {
- List<double[]> prcsInDouleArrayList = new ArrayList<>(pcsInVector.length);
- for (RealVector vector : pcsInVector) prcsInDouleArrayList.add(vector.toArray());
- content.setPrincipalComponents(prcsInDouleArrayList);
- }
- if(model.maximumL2Norm()!=null) content.setMaximumL2Norm(model.maximumL2Norm().toArray());
- if(model.minimumL2Norm()!=null) content.setMinimumL2Norm(model.minimumL2Norm().toArray());
- if(model.statistics()!=null){
- List<Map<String,Object>> statisticsMaps = new ArrayList<>(model.statistics().length);
- for(UserCommandStatistics userCommandStatistics :model.statistics()) statisticsMaps.add(userCommandStatistics.toMap());
- content.setStatistics(statisticsMaps);
- }
-
- entity.setContent(TaggedLogAPIEntity.buildObjectMapper().writeValueAsString(content));
- Map<String,String> tags = new HashMap<>();
- tags.put(UserProfileConstants.SITE_TAG,model.site());
- tags.put(UserProfileConstants.USER_TAG,model.user());
- tags.put(UserProfileConstants.ALGORITHM_TAG,model.algorithm());
-
- entity.setTags(tags);
- return entity;
- }
-
- public static UserProfileEigenModel deserializeModel(MLModelAPIEntity entity) throws IOException {
- if(entity == null) return null;
- String user = entity.getTags() == null ? null: entity.getTags().get(UserProfileConstants.USER_TAG);
- String site = entity.getTags()==null? null:entity.getTags().get(UserProfileConstants.SITE_TAG);
-
- UserProfileEigenModelEntity content = TaggedLogAPIEntity.buildObjectMapper().readValue(entity.getContent(), UserProfileEigenModelEntity.class);
- RealVector[] principalComponents = null;
- if(content.getPrincipalComponents()!=null){
- principalComponents = new RealVector[content.getPrincipalComponents().size()];
- int i = 0;
- for(double[] array:content.getPrincipalComponents()){
- principalComponents[i] = new ArrayRealVector(array);
- i++;
- }
- }
- UserCommandStatistics[] statistics = null;
- if(content.getStatistics()!=null){
- statistics = new UserCommandStatistics[content.getStatistics().size()];
- int i = 0;
- for(Map<String,Object> map:content.getStatistics()){
- statistics[i++] = new UserCommandStatistics(map);
- }
- }
-
- return new UserProfileEigenModel(
- entity.getVersion(),
- site,
- user,
- content.getUMatrix() == null? null : new Array2DRowRealMatrix(content.getUMatrix()),
- content.getDiagonalMatrix() == null ? null : new Array2DRowRealMatrix(content.getDiagonalMatrix()),
- content.getDimension(),
- content.getMinVector() == null ? null : new ArrayRealVector(content.getMinVector()),
- content.getMaxVector() == null? null : new ArrayRealVector(content.getMaxVector()),
- principalComponents,
- content.getMaximumL2Norm() == null ? null : new ArrayRealVector(content.getMaximumL2Norm()),
- content.getMinimumL2Norm() == null ? null : new ArrayRealVector(content.getMinimumL2Norm()),
- statistics
- );
- }
-
- private double[][] uMatrix;
- private double[][] diagonalMatrix;
- private int dimension;
- private double[] minVector;
- private double[] maxVector;
- private List<double[]> principalComponents;
- private double[] maximumL2Norm;
- private double[] minimumL2Norm;
- private List<Map<String,Object>> statistics;
-
- public List<Map<String, Object>> getStatistics() {
- return statistics;
- }
-
- public void setStatistics(List<Map<String, Object>> statistics) {
- this.statistics = statistics;
-// valueChanged("statistics");
- }
-
- public double[][] getUMatrix() {
- return uMatrix;
- }
-
- public void setUMatrix(double[][] uMatrix) {
- this.uMatrix = uMatrix;
-// valueChanged("uMatrix");
- }
-
- public double[][] getDiagonalMatrix() {
- return diagonalMatrix;
- }
-
- public void setDiagonalMatrix(double[][] diagonalMatrix) {
- this.diagonalMatrix = diagonalMatrix;
-// valueChanged("diagonalMatrix");
- }
-
- public int getDimension() {
- return dimension;
- }
-
- public void setDimension(int dimension) {
- this.dimension = dimension;
-// valueChanged("dimension");
- }
-
- public double[] getMinVector() {
- return minVector;
- }
-
- public void setMinVector(double[] minVector) {
- this.minVector = minVector;
- }
-
- public double[] getMaxVector() {
- return maxVector;
- }
-
- public void setMaxVector(double[] maxVector) {
- this.maxVector = maxVector;
- }
-
- public List<double[]> getPrincipalComponents() {
- return principalComponents;
- }
-
- public void setPrincipalComponents(List<double[]> principalComponents) {
- this.principalComponents = principalComponents;
- }
-
- public double[] getMaximumL2Norm() {
- return maximumL2Norm;
- }
-
- public void setMaximumL2Norm(double[] maximumL2Norm) {
- this.maximumL2Norm = maximumL2Norm;
- }
-
- public double[] getMinimumL2Norm() {
- return minimumL2Norm;
- }
-
- public void setMinimumL2Norm(double[] minimumL2Norm) {
- this.minimumL2Norm = minimumL2Norm;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEntityRepository.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEntityRepository.java
deleted file mode 100644
index 88d2d49..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileEntityRepository.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.log.entity.repo.EntityRepository;
-
-public class UserProfileEntityRepository extends EntityRepository {
- public UserProfileEntityRepository(){
- this.registerEntity(ScheduleCommandEntity.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelEntity.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelEntity.java
deleted file mode 100644
index 5209289..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/UserProfileKDEModelEntity.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class UserProfileKDEModelEntity {
- public static UserProfileKDEModel deserializeModel(MLModelAPIEntity entity) throws IOException {
- if(entity == null) return null;
- String user = entity.getTags()==null? null:entity.getTags().get(UserProfileConstants.USER_TAG);
- String site = entity.getTags()==null? null:entity.getTags().get(UserProfileConstants.SITE_TAG);
- UserProfileKDEModelEntity content = TaggedLogAPIEntity.buildObjectMapper().readValue(entity.getContent(), UserProfileKDEModelEntity.class);
-
- UserCommandStatistics[] statistics = null;
- if(content.getStatistics()!=null) {
- statistics = new UserCommandStatistics[content.getStatistics().size()];
- int i = 0;
- for (Map<String, Object> map : content.getStatistics()) {
- statistics[i] = new UserCommandStatistics(map);
- i++;
- }
- }
- return new UserProfileKDEModel(
- entity.getVersion(), site,user,statistics,
- content.getMinProbabilityEstimate(), content.getMaxProbabilityEstimate(),
- content.getNintyFivePercentileEstimate(), content.getMedianProbabilityEstimate());
- }
-
- public static MLModelAPIEntity serializeModel(UserProfileKDEModel model) throws IOException {
- if(model == null) return null;
- MLModelAPIEntity entity = new MLModelAPIEntity();
- UserProfileKDEModelEntity content = new UserProfileKDEModelEntity();
- entity.setVersion(model.version());
- if(model.statistics()!=null){
- List<Map<String,Object>> statisticsMaps = new ArrayList<>(model.statistics().length);
- for(UserCommandStatistics userCommandStatistics :model.statistics()){
- statisticsMaps.add(userCommandStatistics.toMap());
- }
- content.setStatistics(statisticsMaps);
- }
- content.setMinProbabilityEstimate(model.minProbabilityEstimate());
- content.setMaxProbabilityEstimate(model.maxProbabilityEstimate());
- content.setNintyFivePercentileEstimate(model.nintyFivePercentileEstimate());
- content.setMedianProbabilityEstimate(model.medianProbabilityEstimate());
-
- Map<String,String> tags = new HashMap<>();
- tags.put(UserProfileConstants.SITE_TAG,model.site());
- tags.put(UserProfileConstants.USER_TAG,model.user());
- tags.put(UserProfileConstants.ALGORITHM_TAG,model.algorithm());
-
- entity.setTags(tags);
- entity.setContent(TaggedLogAPIEntity.buildObjectMapper().writeValueAsString(content));
- return entity;
- }
-
- private List<Map<String,Object>> statistics;
- private double minProbabilityEstimate;
- private double maxProbabilityEstimate;
- private double nintyFivePercentileEstimate;
- private double medianProbabilityEstimate;
-
- public double getMedianProbabilityEstimate() {
- return medianProbabilityEstimate;
- }
-
- public void setMedianProbabilityEstimate(double medianProbabilityEstimate) {
- this.medianProbabilityEstimate = medianProbabilityEstimate;
- }
-
- public List<Map<String,Object>> getStatistics() {
- return statistics;
- }
-
- public void setStatistics(List<Map<String,Object>> statistics) {
- this.statistics = statistics;
- }
-
- public double getMinProbabilityEstimate() {
- return minProbabilityEstimate;
- }
-
- public void setMinProbabilityEstimate(double minProbabilityEstimate) {
- this.minProbabilityEstimate = minProbabilityEstimate;
- }
-
- public double getMaxProbabilityEstimate() {
- return maxProbabilityEstimate;
- }
-
- public void setMaxProbabilityEstimate(double maxProbabilityEstimate) {
- this.maxProbabilityEstimate = maxProbabilityEstimate;
- }
-
- public double getNintyFivePercentileEstimate() {
- return nintyFivePercentileEstimate;
- }
-
- public void setNintyFivePercentileEstimate(double nintyFivePercentileEstimate) {
- this.nintyFivePercentileEstimate = nintyFivePercentileEstimate;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/package-info.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/package-info.java b/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/package-info.java
deleted file mode 100644
index 20fa623..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/java/org/apache/eagle/security/userprofile/model/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Persistence related model and utilities
- */
-package org.apache.eagle.security.userprofile.model;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/resources/log4j.properties b/eagle-security/eagle-security-userprofile/common/src/main/resources/log4j.properties
deleted file mode 100644
index 3499c46..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/resources/storm.yaml.1
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/resources/storm.yaml.1 b/eagle-security/eagle-security-userprofile/common/src/main/resources/storm.yaml.1
deleted file mode 100644
index 2ea7262..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/resources/storm.yaml.1
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/AuditlogParser.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/AuditlogParser.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/AuditlogParser.scala
deleted file mode 100644
index 47d47d9..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/AuditlogParser.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.eagle.security.hdfs.{HDFSAuditLogObject, HDFSAuditLogParser}
-import org.apache.eagle.security.userprofile.UserProfileUtils
-import org.joda.time.Period
-
-/**
- * {"getfileinfo", "open", "listStatus", "setTimes", "setPermission", "rename", "mkdirs", "create", "setReplication", "contentSummary", "delete", "setOwner", "fsck"}
- */
-case class UserCommandActivity(timestamp:Long,user:String,cmd:String,periodWindowSeconds:Long)
-
-/**
- * SELECT a.ugi, a.dt, a.hour, a.cmd, count(1)
- * from audit_log a
- * WHERE a.dt>='${STARTDATE}' AND a.dt<='${ENDDATE}'AND a.hour >= '${STARTTIME}' AND a.hour < '${ENDTIME}'
- * GROUP BY a.ugi,a.dt, a.hour, a.cmd;
- */
-case class AuditLogTransformer(period:Period) extends Serializable {
- val parser = new HDFSAuditLogParser()
- // serialize periodInSeconds to avoid recreating Seconds object
- val periodInSeconds = period.toStandardSeconds
-
- def getObject(value: String) : Option[HDFSAuditLogObject] = {
- Option(parser.parse(value))
- }
-
- def transform(value: String) : Option[UserCommandActivity] = {
- Option(parser.parse(value)) match {
- case Some(entity) =>
- val timestamp = entity.timestamp
- val user = entity.user
- val cmd = entity.cmd
- val periodWindowSeconds = UserProfileUtils.formatSecondsByPeriod (timestamp / 1000, periodInSeconds)
- Some(UserCommandActivity (timestamp, user, cmd, periodWindowSeconds))
- case None => None
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/EntityConversion.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/EntityConversion.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/EntityConversion.scala
deleted file mode 100644
index 266bf88..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/EntityConversion.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity
-
-/**
- * @since 7/22/15
- */
-private[userprofile] trait EntityConversion[M <: TaggedLogAPIEntity] {
- def toEntity: M
- def fromEntity (m: M)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserActivityAggModel.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserActivityAggModel.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserActivityAggModel.scala
deleted file mode 100644
index edd5aa7..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserActivityAggModel.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.commons.math3.linear.RealMatrix
-
-/**
- * @since 7/31/15
- */
-case class UserActivityAggModel(user:String,matrix:RealMatrix,cmdTypes:Seq[String],site:String,timestamp:Long)
- extends EntityConversion[UserActivityAggModelEntity]{
- override def toEntity: UserActivityAggModelEntity = UserActivityAggModelEntity.fromModel(this)
- override def fromEntity(entity: UserActivityAggModelEntity): Unit = UserActivityAggModelEntity.toModel(entity)
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileEigenModel.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileEigenModel.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileEigenModel.scala
deleted file mode 100644
index cdb4205..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileEigenModel.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.commons.math3.linear.{RealMatrix, RealVector}
-import org.apache.eagle.ml.model.MLModelAPIEntity
-import org.apache.eagle.security.userprofile.UserProfileConstants
-import org.codehaus.jackson.map.annotate.JsonSerialize
-
-/**
- * @since 7/22/15
- */
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-case class UserProfileEigenModel(
- override val version:Long,
- override val site:String,
- override val user:String,
- uMatrix: RealMatrix,
- diagonalMatrix: RealMatrix,
- dimension: Int,
- minVector: RealVector,
- maxVector: RealVector,
- principalComponents: Array[RealVector],
- maximumL2Norm:RealVector,
- minimumL2Norm:RealVector,
- statistics:Array[UserCommandStatistics]
-) extends UserProfileModel(site,user,UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM,version) with EntityConversion[MLModelAPIEntity] {
- override def toEntity: MLModelAPIEntity = UserProfileEigenModelEntity.serializeModel(this)
- override def fromEntity(m: MLModelAPIEntity): Unit = UserProfileEigenModelEntity.deserializeModel(m)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileKDEModel.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileKDEModel.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileKDEModel.scala
deleted file mode 100644
index bd0af47..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileKDEModel.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.eagle.ml.model.MLModelAPIEntity
-import org.apache.eagle.security.userprofile.UserProfileConstants
-
-/**
- * @since 7/22/15
- */
-case class UserProfileKDEModel (
- override val version:Long,
- override val site:String,
- override val user:String,
- statistics: Array[UserCommandStatistics],
- minProbabilityEstimate:Double,
- maxProbabilityEstimate: Double,
- nintyFivePercentileEstimate: Double,
- medianProbabilityEstimate: Double
-) extends UserProfileModel(site,user,UserProfileConstants.KDE_ALGORITHM,version ) with EntityConversion[MLModelAPIEntity]{
- override def toEntity: MLModelAPIEntity = UserProfileKDEModelEntity.serializeModel(this)
- override def fromEntity(m: MLModelAPIEntity): Unit = UserProfileKDEModelEntity.deserializeModel(m)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModel.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModel.scala b/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModel.scala
deleted file mode 100644
index decb0ed..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModel.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-/**
- * @since 7/21/15
- */
-abstract class UserProfileModel(val site:String,val user:String,val algorithm:String, val version:Long)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/test/java/eagle/auditlog/userprofile/common/TestUserProfileUtils.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/test/java/eagle/auditlog/userprofile/common/TestUserProfileUtils.java b/eagle-security/eagle-security-userprofile/common/src/test/java/eagle/auditlog/userprofile/common/TestUserProfileUtils.java
deleted file mode 100644
index d98907f..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/test/java/eagle/auditlog/userprofile/common/TestUserProfileUtils.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.auditlog.userprofile.common;
-
-import org.apache.eagle.security.userprofile.UserProfileUtils;
-import org.apache.eagle.common.DateTimeUtil;
-import org.joda.time.Period;
-import org.joda.time.Seconds;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.text.ParseException;
-
-public class TestUserProfileUtils {
- @Test
- public void testJodaTimePeriod() throws ParseException {
- String periodText = "PT10m";
- Period period = new Period(periodText);
- int seconds = period.toStandardSeconds().getSeconds();
- Assert.assertEquals(600, seconds);
- Assert.assertEquals(60, period.toStandardSeconds().dividedBy(10).getSeconds());
- }
-
- @Test
- public void testFormatSecondsByPeriod15M() throws ParseException {
-
- Period period = new Period("PT15m");
- Seconds seconds = period.toStandardSeconds();
- Assert.assertEquals(15*60,seconds.getSeconds());
-
- long time = DateTimeUtil.humanDateToSeconds("2015-07-01 13:56:12");
- long expect = DateTimeUtil.humanDateToSeconds("2015-07-01 13:45:00");
- long result = UserProfileUtils.formatSecondsByPeriod(time,seconds);
- Assert.assertEquals(expect,result);
-
- time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
- expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- Assert.assertEquals(expect,result);
-
- time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
- expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- Assert.assertEquals(expect,result);
- }
-
- @Test
- public void testFormatSecondsByPeriod1H() throws ParseException {
-
- Period period = new Period("PT1h");
- Seconds seconds = period.toStandardSeconds();
- Assert.assertEquals(60*60,seconds.getSeconds());
-
- long time = DateTimeUtil.humanDateToSeconds("2015-07-01 13:56:12");
- long expect = DateTimeUtil.humanDateToSeconds("2015-07-01 13:00:00");
- long result = UserProfileUtils.formatSecondsByPeriod(time,seconds);
- Assert.assertEquals(expect,result);
-
- time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
- expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- Assert.assertEquals(expect,result);
-
- time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:30:59");
- expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- Assert.assertEquals(expect,result);
- }
-
- /*@Test
- public void testFormatSecondsByPeriod1Min() throws ParseException {
-
- Period period = new Period("PT1M");
- Seconds seconds = period.toStandardSeconds();
- Assert.assertEquals(60,seconds.getSeconds());
-
- long time = DateTimeUtil.humanDateToSeconds("2015-07-01 13:56:15");
- long expect = DateTimeUtil.humanDateToSeconds("2015-07-01 13:55:16");
- long result = UserProfileUtils.formatSecondsByPeriod(time,seconds);
- Assert.assertEquals(expect,result);
-
- //time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:14:59");
- //expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- //result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- //Assert.assertEquals(expect,result);
-
- //time = DateTimeUtil.humanDateToSeconds("2015-07-01 03:30:59");
- //expect = DateTimeUtil.humanDateToSeconds("2015-07-01 03:00:00");
- //result = UserProfileUtils.formatSecondsByPeriod(time, seconds);
- //Assert.assertEquals(expect,result);
- }*/
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/test/resources/log4j.properties b/eagle-security/eagle-security-userprofile/common/src/test/resources/log4j.properties
deleted file mode 100644
index 3499c46..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/common/src/test/resources/storm.yaml.2
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/common/src/test/resources/storm.yaml.2 b/eagle-security/eagle-security-userprofile/common/src/test/resources/storm.yaml.2
deleted file mode 100644
index 2ea7262..0000000
--- a/eagle-security/eagle-security-userprofile/common/src/test/resources/storm.yaml.2
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-metadata.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-metadata.sh b/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-metadata.sh
deleted file mode 100755
index 5b1891a..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-metadata.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-export SERVICE_HOST="localhost"
-export SERVICE_PORT=9099
-
-echo "Creating alert definition for AlertDefinitionService with userprofile-policy-definition.json"
-result=$(curl -XPOST -H "Content-Type: application/json" \
- "http://$SERVICE_HOST:$SERVICE_PORT/eagle-service/rest/entities?serviceName=AlertDefinitionService" \
- -d '[ { "prefix": "alertdef", "tags": { "site": "sandbox", "dataSource": "userProfile", "alertExecutorId": "userProfileAnomalyDetectionExecutor", "policyId": "userProfile", "policyType": "MachineLearning" }, "desc": "user profile anomaly detection", "policyDef": "{\"type\":\"MachineLearning\",\"alertContext\":{\"site\":\"sandbox\",\"dataSource\":\"userProfile\",\"component\":\"testComponent\",\"description\":\"ML based user profile anomaly detection\",\"severity\":\"WARNING\",\"notificationByEmail\":\"true\"},\"algorithms\":[{\"name\":\"EigenDecomposition\",\"evaluator\":\"eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator\",\"description\":\"EigenBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"},{\"name\":\"KDE\",\"evaluator\":\"eagle.security.userprofile.impl.UserProfileAnomalyKDEEvaluator\",\"description\":\"DensityBasedAnomalyDetect
ion\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"}]}", "dedupeDef": "{\"alertDedupIntervalMin\":\"0\",\"emailDedupIntervalMin\":\"0\"}", "notificationDef": "", "remediationDef": "", "enabled": true } ]' 2>/dev/null)
-
-echo "=> $result"
-
-echo ""
-
-echo "Creating alert executor for AlertExecutorService with userprofile-executor-definition.json"
-result=$(curl -XPOST -H "Content-Type: application/json" \
- "http://$SERVICE_HOST:$SERVICE_PORT/eagle-service/rest/entities?serviceName=AlertExecutorService" \
- -d '[ { "prefix": "alertExecutor", "tags":{ "site":"sandbox", "dataSource":"userProfile", "alertExecutorId" : "userProfileAnomalyDetectionExecutor", "streamName":"userActivity" }, "desc": "user activity data source" } ]' 2>/dev/null)
-
-echo "=> $result"
-
-echo ""
-
-echo "Creating alert executor for AlertStreamService with userprofile-stream-description.json"
-result=$(curl -XPOST -H "Content-Type: application/json" \
- "http://$SERVICE_HOST:$SERVICE_PORT/eagle-service/rest/entities?serviceName=AlertStreamService" \
- -d '[ { "prefix": "alertStream", "tags": { "streamName": "userActivity", "site":"sandbox", "dataSource":"userProfile" }, "alertExecutorIdList": [ "userProfileAnomalyDetectionExecutor" ] } ]' 2>/dev/null)
-
-echo "=> $result"
[6/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-table.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-table.sh b/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-table.sh
deleted file mode 100644
index 862f836..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/prepare-table.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-create 'alertStreamSchema', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
-create 'mlmodel', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
-create 'eagle_metric', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
-create 'userprofile', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity-with-spark.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity-with-spark.sh b/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity-with-spark.sh
deleted file mode 100644
index fbc540d..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity-with-spark.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-spark-submit --master yarn-cluster --deploy-mode cluster --class eagle.security.userprofile.UserProfileTrainingCLI eagle-security-userprofile-training-0.1.0-assembly.jar --master yarn-cluster --period PT1M --input /tmp/auditlog/* --kafka-props topic=sandbox_hdfs_audit_agg,metadata.broker.list=sandbox.hortonworks.com:6667
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity.sh b/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity.sh
deleted file mode 100644
index 6d4f0b1..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/produce-useractivity.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-/usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list sandbox.hortonworks.com:6667 --topic sandbox_hdfs_audit_agg
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/useractivity-agg-json.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/useractivity-agg-json.txt b/eagle-security/eagle-security-userprofile/detection/dev-supports/useractivity-agg-json.txt
deleted file mode 100644
index 0d0268c..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/useractivity-agg-json.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-{"timestamp":1439462433990,"tags":{"user":"oozie"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462434293,"tags":{"user":"root"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[4.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462433990,"tags":{"user":"mapred"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[0.0,0.0,39.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,66.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,78.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,69.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,73.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462434293,"tags":{"user":"hchen9"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462434294,"tags":{"user":"hue"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[400.0,4.0,77.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[21.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462434294,"tags":{"user":"hdfs"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439462434294,"tags":{"user":"hbase"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[33.0,3.0,31.0,0.0,0.0,2.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0]]}
-{"timestamp":1439802310049,"tags":{"user":"root"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[4.0,0.0,0.0,0.0,0.0,1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0],[2.0,35.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[155.0,6.0,27.0,0.0,0.0,49.0,2.0,25.0,0.0,0.0,1.0,0.0,0.0],[148.0,3.0,25.0,0.0,0.0,48.0,1.0,25.0,0.0,0.0,1.0,0.0,0.0],[639.0,30.0,100.0,0.0,11.0,195.0,12.0,109.0,6.0,0.0,7.0,0.0,0.0],[1242.0,33.0,202.0,0.0,12.0,390.0,18.0,212.0,8.0,0.0,12.0,0.0,0.0],[650.0,32.0,100.0,0.0,12.0,196.0,12.0,112.0,8.0,0.0,8.0,0.0,0.0],[45.0,9.0,7.0,0.0,3.0,9.0,4.0,9.0,2.0,0.0,3.0,0.0,0.0],[181.0,32.0,19.0,0.0,15.0,35.0,25.0,30.0,10.0,0.0,10.0,0.0,0.0],[28.0,4.0,0.0,0.0,6.0,2.0,4.0,6.0,4.0,0.0,1.0,0.0,0.0],[14.0,2.0,0.
0,0.0,3.0,1.0,2.0,3.0,2.0,0.0,1.0,0.0,0.0],[15.0,2.0,0.0,0.0,3.0,1.0,2.0,3.0,2.0,0.0,1.0,0.0,0.0],[14.0,2.0,0.0,0.0,3.0,1.0,2.0,3.0,2.0,0.0,1.0,0.0,0.0],[14.0,2.0,0.0,0.0,3.0,1.0,2.0,3.0,2.0,0.0,1.0,0.0,0.0],[14.0,2.0,0.0,0.0,3.0,1.0,2.0,3.0,2.0,0.0,1.0,0.0,0.0],[161.0,38.0,0.0,0.0,30.0,12.0,22.0,40.0,20.0,0.0,7.0,0.0,0.0],[100.0,36.0,0.0,0.0,18.0,6.0,14.0,34.0,12.0,0.0,4.0,0.0,0.0],[18.0,8.0,0.0,0.0,3.0,1.0,3.0,11.0,2.0,0.0,1.0,0.0,0.0]]}
-{"timestamp":1439802310151,"tags":{"user":"yarn"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[3.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[5.0,0.0,5.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[6.0,0.0,6.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313098,"tags":{"user":"spark"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313149,"tags":{"user":"mapred"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,66.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,78.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,69.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,78.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,49.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,41.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,15.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],
[0.0,0.0,74.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,81.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,71.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,45.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,77.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,27.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,71.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,76.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,61.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,67.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,73.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,73.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0
,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,76.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,45.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,48.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,20.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,24.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,22.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,22.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,40.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,67.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,64.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,48.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,76.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,26.0,0.0
,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,53.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,67.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,64.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,69.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,74.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,35.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,76.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,18.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,30.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,41.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,50.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,38.0,0.0,0.0,0.0,0
.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,23.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,44.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,68.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,57.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,40.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,57.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,1.0,47.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[23.0,23.0,52.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,39.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,75.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,44.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,48.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[5.0,0.0,40.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,68.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,58.0,0.0,0.0,0.0,0.0,0.
0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,76.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,41.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,72.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,74.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,77.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,48.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,58.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,27.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,78.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,45.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,32.0,0.0,0.0,0.0,0.0,0.0,0.0,0
.0,0.0,0.0,0.0],[0.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,49.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,79.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,23.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,17.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,68.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,74.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,20.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,43.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,28.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,80.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,41.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[8.0,8.0,26.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,20.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,66.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,1.0,40.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0
.0,0.0],[2.0,2.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,8.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,35.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,33.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,38.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[3.0,3.0,27.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,4.0,39.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,7.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,1.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,7.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,22.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,24.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.
0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,19.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,17.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,20.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,27.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,30.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,21.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,22.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,43.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,34.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,22.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[3.0,1.0,34.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,2.0,56.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,15.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313192,"tags":{"user":"hdfs"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[3.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[15.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[5.0,0.0,0.0,0.0,0.0,0.0,1.0,2.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],
[2.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[14.0,0.0,7.0,0.0,0.0,0.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[8.0,0.0,6.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,2.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0
.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[7.0,0.0,7.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[4.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[2.0,0.0,0.0,0.0,0.
0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[3.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0],[11.0,0.0,12.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313394,"tags":{"user":"oozie"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,54.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[
0.0,0.0,14.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,62.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,53.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,19.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,
60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,45.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,41.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,18.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,45.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,62.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,27.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,8.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,
0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,44.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,15.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,61.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,25.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,51.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,30.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0
.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,62.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,64.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,68.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,29.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,28.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,47.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,25.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,63.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,
0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,64.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,30.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,58.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
,0.0,0.0,0.0],[0.0,0.0,36.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,20.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,55.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,47.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,23.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,31.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,21.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,30.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,50.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,37.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,51.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,70.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.
0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,23.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,48.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,49.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,42.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,47.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,60.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,59.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313648,"tags":{"user":"hue"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[400.0,4.0,77.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[32.0,0.0,6.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[56.0,20.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[40.0,6.0,5.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[142.0,22.0,18.0,0.0,0.0,1.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0],[100.0,44.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[157.0,36.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[173.0,6.0,29.0,0.0,0.0,6.0,1.0,3.0,0.0,0.0,2.0,0.0,0.0],[68.0,4.0,10.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[226.0,10.0,39.0,0.0,0.0,6.0,0.0,3.0,0.0,0.0,3.0,0.0,0.0],[60.0,0.0,12.0,0.0,0.0,5.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0],[60.0,2.0,10.0,0.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[10.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0],[60.0,2.0,10.0,0.0,0.0,1.0,0.0,1.0,0.0,
0.0,0.0,0.0,0.0],[36.0,0.0,7.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[110.0,18.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[87.0,8.0,14.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313697,"tags":{"user":"hchen9"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[3.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[3.0,17.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]]}
-{"timestamp":1439802313747,"tags":{"user":"hbase"},"cmdTypes":["getfileinfo","open","listStatus","setTimes","setPermission","rename","mkdirs","create","setReplication","contentSummary","delete","setOwner","fsck"],"cmdMatrix":[[76.0,24.0,148.0,6.0,0.0,10.0,4.0,4.0,0.0,0.0,14.0,0.0,0.0],[22.0,3.0,9.0,3.0,0.0,5.0,2.0,3.0,0.0,0.0,2.0,0.0,0.0],[111.0,9.0,21.0,0.0,0.0,0.0,2.0,0.0,0.0,0.0,5.0,0.0,0.0],[17.0,3.0,23.0,0.0,0.0,1.0,2.0,2.0,0.0,0.0,2.0,0.0,0.0],[24.0,0.0,120.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[190.0,78.0,244.0,5.0,0.0,7.0,8.0,16.0,0.0,0.0,25.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[28.0,0.0,100.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[58.0,3.0,17.0,1.0,0.0,3.0,2.0,2.0,0.0,0.0,2.0,0.0,0.0],[15.0,3.0,5.0,0.0,0.0,1.0,2.0,2.0,0.0,0.0,2.0,0.0,0.0],[358.0,125.0,308.0,7.0,0.0,12.0,13.0,43.0,0.0,0.0,43.0,0.0,0.0],[35.0,0.0,128.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,10.0,0.0,0.0],[32.0,0.0,120.0,2.0,0
.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[48.0,4.0,132.0,2.0,0.0,6.0,6.0,8.0,0.0,0.0,2.0,0.0,0.0],[40.0,12.0,120.0,1.0,0.0,3.0,1.0,4.0,0.0,0.0,1.0,0.0,0.0],[34.0,0.0,120.0,3.0,0.0,3.0,0.0,2.0,0.0,0.0,3.0,0.0,0.0],[30.0,0.0,120.0,1.0,0.0,1.0,0.0,2.0,0.0,0.0,1.0,0.0,0.0],[116.0,20.0,244.0,8.0,0.0,14.0,7.0,9.0,0.0,0.0,21.0,0.0,0.0],[9.0,6.0,26.0,0.0,0.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0],[2.0,4.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[212.0,81.0,189.0,3.0,0.0,7.0,10.0,20.0,0.0,0.0,22.0,0.0,0.0],[26.0,0.0,84.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[412.0,115.0,263.0,5.0,0.0,10.0,12.0,21.0,0.0,0.0,29.0,0.0,0.0],[40.0,12.0,120.0,1.0,0.0,3.0,2.0,4.0,0.0,0.0,1.0,0.0,0.0],[34.0,0.0,120.0,3.0,0.0,3.0,0.0,2.0,0.0,0.0,3.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[62.0,12.0,168.0,4.0,0.0,6.0,1.0,4.0,0.0,0.0,9.0,0.0,0.0],[37.0,6.0,120.0,2.0,0.0,3.0,0.0,3.0,0.0,0.0,2.0,0.0,0.0],[34.0,0.0,120.0,3.0,0.0,3.0,0.0,2.0,
0.0,0.0,3.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[26.0,0.0,90.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[71.0,3.0,17.0,2.0,0.0,4.0,2.0,3.0,0.0,0.0,2.0,0.0,0.0],[33.0,3.0,89.0,0.0,0.0,1.0,2.0,2.0,0.0,0.0,4.0,0.0,0.0],[24.0,0.0,120.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[33.0,3.0,31.0,0.0,0.0,2.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0],[205.0,85.0,245.0,5.0,0.0,8.0,10.0,18.0,0.0,0.0,25.0,0.0,0.0],[62.0,12.0,168.0,4.0,0.0,6.0,2.0,4.0,0.0,0.0,9.0,0.0,0.0],[32.0,0.0,108.0,3.0,0.0,3.0,0.0,2.0,0.0,0.0,3.0,0.0,0.0],[40.0,3.0,17.0,1.0,0.0,3.0,2.0,2.0,0.0,0.0,2.0,0.0,0.0],[167.0,71.0,157.0,2.0,0.0,4.0,9.0,17.0,0.0,0.0,19.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[217.0,84.0,231.0,5.0,0.0,10.0,12.0,19.0,0.0,0.0,27.0,0.0,0.0]
,[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[14.0,0.0,38.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,0.0,0.0,0.0],[4.0,4.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.0,0.0,0.0],[162.0,71.0,140.0,2.0,0.0,4.0,8.0,16.0,0.0,0.0,16.0,0.0,0.0],[34.0,12.0,96.0,1.0,0.0,3.0,2.0,4.0,0.0,0.0,1.0,0.0,0.0],[3665.0,3.0,17.0,0.0,0.0,2.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0],[330.0,100.0,181.0,6.0,0.0,21.0,12.0,30.0,0.0,0.0,40.0,0.0,0.0],[12.0,0.0,62.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[275.0,90.0,98.0,2.0,0.0,15.0,8.0,27.0,0.0,0.0,25.0,0.0,0.0],[31.0,0.0,120.0,1.0,0.0,1.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[96.0,24.0,216.0,9.0,0.0,13.0,4.0,6.0,0.0,0.0,17.0,0.0,0.0],[33.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,3.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[32.0,0.0,120.0,2.0,0.0,2.0,0.0,2.0,0.0,0.0,2.0,0.0,0.0],[16.0,0.0,62.
0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.0,0.0,0.0],[461.0,163.0,256.0,7.0,0.0,25.0,19.0,45.0,0.0,0.0,55.0,0.0,0.0],[40.0,12.0,120.0,1.0,0.0,3.0,2.0,4.0,0.0,0.0,1.0,0.0,0.0],[96.0,24.0,208.0,8.0,0.0,12.0,2.0,6.0,0.0,0.0,16.0,0.0,0.0],[34.0,0.0,120.0,3.0,0.0,3.0,0.0,2.0,0.0,0.0,3.0,0.0,0.0],[33.0,6.0,37.0,0.0,0.0,2.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0],[168.0,81.0,109.0,5.0,0.0,8.0,10.0,18.0,0.0,0.0,17.0,0.0,0.0],[56.0,12.0,160.0,1.0,0.0,3.0,2.0,4.0,0.0,0.0,10.0,0.0,0.0],[30.0,0.0,110.0,3.0,0.0,3.0,0.0,2.0,0.0,0.0,1.0,0.0,0.0],[2.0,4.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],[21.0,7.0,23.0,0.0,0.0,0.0,1.0,1.0,0.0,0.0,4.0,0.0,0.0],[97.0,4.0,13.0,0.0,0.0,2.0,3.0,2.0,0.0,0.0,2.0,0.0,0.0],[170.0,79.0,124.0,3.0,0.0,6.0,11.0,17.0,0.0,0.0,21.0,0.0,0.0],[30.0,0.0,120.0,1.0,0.0,1.0,0.0,2.0,0.0,0.0,1.0,0.0,0.0],[59.0,3.0,16.0,0.0,0.0,1.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0],[15.0,3.0,5.0,0.0,0.0,1.0,2.0,2.0,0.0,0.0,2.0,0.0,0.0],[57.0,3.0,15.0,0.0,0.0,1.0,2.0,1.0,0.0,0.0,2.0,0.0,0.0],[116.0,0.0,1.0,0.0,0.0,0.0,0.
0,0.0,0.0,0.0,0.0,0.0,0.0]]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-executor-definition.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-executor-definition.json b/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-executor-definition.json
deleted file mode 100644
index b2f5f00..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-executor-definition.json
+++ /dev/null
@@ -1 +0,0 @@
-[ { "prefix": "alertExecutor", "tags":{ "site":"sandbox", "dataSource":"userProfile", "alertExecutorId" : "userProfileAnomalyDetectionExecutor", "streamName":"userActivity" }, "desc": "user activity data source" } ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-policy-definition.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-policy-definition.json b/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-policy-definition.json
deleted file mode 100644
index 36ed57a..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-policy-definition.json
+++ /dev/null
@@ -1 +0,0 @@
-[ { "prefix": "alertdef", "tags": { "site": "sandbox", "dataSource": "userProfile", "alertExecutorId": "userProfileAnomalyDetectionExecutor", "policyId": "userProfile", "policyType": "MachineLearning" }, "desc": "user profile anomaly detection", "policyDef": "{\"type\":\"MachineLearning\",\"alertContext\":{\"site\":\"sandbox\",\"dataSource\":\"userProfile\",\"component\":\"testComponent\",\"description\":\"ML based user profile anomaly detection\",\"severity\":\"WARNING\",\"notificationByEmail\":\"true\"},\"algorithms\":[{\"name\":\"EigenDecomposition\",\"evaluator\":\"eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator\",\"description\":\"EigenBasedAnomalyDetection\",\"features\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"},{\"name\":\"KDE\",\"evaluator\":\"eagle.security.userprofile.impl.UserProfileAnomalyKDEEvaluator\",\"description\":\"DensityBasedAnomalyDetection\",\"feat
ures\":\"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck\"}]}", "dedupeDef": "{\"alertDedupIntervalMin\":\"0\",\"emailDedupIntervalMin\":\"0\"}", "notificationDef": "", "remediationDef": "", "enabled": true } ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-description.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-description.json b/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-description.json
deleted file mode 100644
index 9977bde..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-description.json
+++ /dev/null
@@ -1 +0,0 @@
-[ { "prefix": "alertStream", "tags": { "streamName": "userActivity", "site":"sandbox", "dataSource":"userProfile" }, "alertExecutorIdList": [ "userProfileAnomalyDetectionExecutor" ] } ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-metadata.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-metadata.json b/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-metadata.json
deleted file mode 100644
index a630f9b..0000000
--- a/eagle-security/eagle-security-userprofile/detection/dev-supports/userprofile-stream-metadata.json
+++ /dev/null
@@ -1 +0,0 @@
-[ { "prefix": "streamMetadata", "tags":{ "site" : "sandbox", "dataSource":"userProfile", "streamName" : "userActivity", "attrName" : "user" }, "attrType": "string", "category": "", "attrValueResolver" : "" } ]
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/pom.xml b/eagle-security/eagle-security-userprofile/detection/pom.xml
deleted file mode 100644
index 512b8b1..0000000
--- a/eagle-security/eagle-security-userprofile/detection/pom.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-userprofile-parent</artifactId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <artifactId>eagle-security-userprofile-detection</artifactId>
- <packaging>jar</packaging>
- <name>eagle-security-userprofile-detection</name>
- <url>http://maven.apache.org</url>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-userprofile-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-alert-process</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm-commons</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm-tree</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-machinelearning-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-hdfs-auditlog</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala-lang.version}</version>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/run_onlineprediction_topology.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/run_onlineprediction_topology.sh b/eagle-security/eagle-security-userprofile/detection/run_onlineprediction_topology.sh
deleted file mode 100755
index 43f44ae..0000000
--- a/eagle-security/eagle-security-userprofile/detection/run_onlineprediction_topology.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export JAVA_HOME=$(/usr/libexec/java_home -v 1.6)
-mvn -X exec:java -Dexec.mainClass="eagle.security.userprofile.UserProfileDetectionAnomalyRunner"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/TimeWindow.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/TimeWindow.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/TimeWindow.java
deleted file mode 100644
index c8ea352..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/TimeWindow.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-/**
- * Aggregation time window
- *
- * @since 9/29/15
- */
-public interface TimeWindow{
- /**
- * @return start time stamp
- */
- long from();
-
- /**
- * @return stop timestamp
- */
- long to();
-
- /**
- * @param timestamp income event timestamp
- * @return whether accept the event
- */
- boolean accept(Long timestamp);
-
- /**
- * @return whether current time-window is expired
- */
- boolean expire();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserActivityAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserActivityAggregator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserActivityAggregator.java
deleted file mode 100644
index 7e3b867..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserActivityAggregator.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import scala.Tuple2;
-
-import java.util.Map;
-
-/**
- * @since 9/28/15
- */
-public interface UserActivityAggregator {
- /**
- * @param event accumulated data point
- * @param collector output collector
- */
- void accumulate(Map<String,Object> event,Collector<Tuple2<String, UserActivityAggModelEntity>> collector);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAggregatorExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAggregatorExecutor.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAggregatorExecutor.java
deleted file mode 100644
index 4dd4c7f..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAggregatorExecutor.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import com.typesafe.config.Config;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.datastream.JavaStormStreamExecutor2;
-import org.apache.eagle.security.userprofile.impl.UserActivityAggregatorImpl;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @since 9/29/15
- */
-public class UserProfileAggregatorExecutor extends JavaStormStreamExecutor2<String, UserActivityAggModelEntity> {
- private static final Logger LOG = LoggerFactory.getLogger(UserProfileAggregatorExecutor.class);
- private UserActivityAggregator aggregator;
- private String site;
- private String granularity;
- private long safeWindowMs = 5000l; // 5s
-
- private String[] cmdFeatures = UserProfileConstants.DEFAULT_CMD_TYPES;
-
- @Override
- public void prepareConfig(Config config) {
- this.site = config.getString("eagleProps.site");
- if(config.hasPath("eagleProps.userProfileGranularity")){
- this.granularity = config.getString("eagleProps.userProfileGranularity");
- LOG.info("Override [userProfileGranularity] = [PT1m]");
- }else{
- LOG.info("Using default [userProfileGranularity] = [PT1m]");
- this.granularity = "PT1m";
- }
- if(config.hasPath("eagleProps.userProfileSafeWindowMs")){
- this.safeWindowMs = config.getLong("eagleProps.userProfileSafeWindowMs");
- }
- if(config.hasPath("eagleProps.userProfileCmdFeatures")){
- String cmdFeaturesStr = config.getString("eagleProps.userProfileCmdFeatures");
- LOG.info(String.format("Override [userProfileCmdFeatures] = [%s]",cmdFeaturesStr));
- this.cmdFeatures = cmdFeaturesStr.split(",");
- }else{
- LOG.info(String.format("Using default [userProfileCmdFeatures] = [%s]", StringUtils.join(this.cmdFeatures,",")));
- }
- }
-
- @Override
- public void init() {
- aggregator = new UserActivityAggregatorImpl(Arrays.asList(cmdFeatures),Period.parse(this.granularity),site,this.safeWindowMs);
- }
-
- @Override
- public void flatMap(List<Object> input, Collector<Tuple2<String, UserActivityAggModelEntity>> collector) {
- String user = (String) input.get(0);
- Map<String,Object> event = (Map<String, Object>) input.get(1);
- aggregator.accumulate(event,collector);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAnomalyDetector.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAnomalyDetector.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAnomalyDetector.java
deleted file mode 100644
index d8c65af..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileAnomalyDetector.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserProfileModel;
-
-import java.io.IOException;
-import java.util.List;
-
-public interface UserProfileAnomalyDetector<M extends UserProfileModel> {
- /**
- * Convert from entity to model
- *
- * @param entity model entity
- * @return model in type of M
- */
- M convert(MLModelAPIEntity entity) throws IOException;
-
- /**
- * @param user user name
- * @param algorithm algorithm name
- * @param userActivity real-time user activity aggregation entity
- * @param model user profiling model
- * @return MLAnomalyCallback list
- */
- List<MLCallbackResult> detect(String user, String algorithm, UserActivityAggModel userActivity, M model);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
deleted file mode 100644
index c39f8fb..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionBatchMain.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.security.userprofile;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.datastream.*;
-import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.Map;
-
-public class UserProfileDetectionBatchMain {
- public static void main(String[] args) throws Exception{
- StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
- env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1)
- .flatMap(new UserActivityPartitionExecutor())
- .alertWithConsumer(UserProfileDetectionConstants.USER_ACTIVITY_AGGREGATION_STREAM,
- UserProfileDetectionConstants.USER_PROFILE_ANOMALY_DETECTION_EXECUTOR);
- env.execute();
- }
-
- public static class UserActivityPartitionExecutor extends JavaStormStreamExecutor2<String, Map> {
- private final static Logger LOG = LoggerFactory.getLogger(UserActivityPartitionExecutor.class);
- @Override
- public void prepareConfig(Config config) {
- // do nothing
- }
-
- @Override
- public void init() {
- // do nothing
- }
-
- @Override
- public void flatMap(java.util.List<Object> input, Collector<Tuple2<String, Map>> outputCollector){
- if(input.size()>0){
- Object obj = input.get(0);
- if(obj instanceof UserActivityAggModelEntity){
- UserActivityAggModelEntity entity = (UserActivityAggModelEntity) obj;
- String user = entity.getTags() == null? null: entity.getTags().get(UserProfileConstants.USER_TAG);
- outputCollector.collect(new Tuple2(user,entity));
- }else{
- LOG.warn(String.format("%s is not instance of UserActivityAggModelEntity, skip",obj));
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionConstants.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionConstants.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionConstants.java
deleted file mode 100644
index d4b275b..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionConstants.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-/**
- * @since 9/29/15
- */
-public class UserProfileDetectionConstants {
- public final static String USER_PROFILE_ANOMALY_DETECTION_EXECUTOR = "userProfileAnomalyDetectionExecutor";
- public final static String USER_ACTIVITY_AGGREGATION_STREAM = "userActivity";
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionMain.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionMain.java
deleted file mode 100644
index 61449d2..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionMain.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.eagle.security.userprofile;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class UserProfileDetectionMain {
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileDetectionMain.class);
- public final static String USERPROFILE_DETECTION_MODE_KEY="eagleProps.userProfileMode";
- public static void main(String[] args) throws Exception {
- Config config = new ConfigOptionParser().load(args);
- LOG.info("Config class: " + config.getClass().getCanonicalName());
-
- if(config.hasPath(USERPROFILE_DETECTION_MODE_KEY) && config.getString(USERPROFILE_DETECTION_MODE_KEY).equalsIgnoreCase("batch")){
- LOG.info("Starting UserProfileDetection Topology in [batch] mode");
- UserProfileDetectionBatchMain.main(args);
- }else{
- LOG.info("Starting UserProfileDetection Topology in [streaming] mode");
- UserProfileDetectionStreamMain.main(args);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
deleted file mode 100644
index 8b82ef7..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileDetectionStreamMain.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.security.userprofile;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.datastream.*;
-import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
-import scala.Tuple2;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-public class UserProfileDetectionStreamMain {
- public static void main(String[] args) throws Exception{
- StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
- env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
- .flatMap(new AuditLogTransformer()).nameAs("transformer") // [user,map]
- .groupBy(Arrays.asList(0)) // group by [user]
- .flatMap(new UserProfileAggregatorExecutor()).nameAs("aggregator")
- .alertWithConsumer(UserProfileDetectionConstants.USER_ACTIVITY_AGGREGATION_STREAM,
- UserProfileDetectionConstants.USER_PROFILE_ANOMALY_DETECTION_EXECUTOR); // alert
- env.execute();
- }
-
- private static class AuditLogTransformer extends JavaStormStreamExecutor2<String,Map<String,Object>> {
- @Override
- public void prepareConfig(Config config) {}
-
- @Override
- public void init() {}
-
- @Override
- public void flatMap(List<Object> input, Collector<Tuple2<String, Map<String, Object>>> collector) {
- Map<String, Object> auditLog = (Map<String, Object>) input.get(0);
- collector.collect(new Tuple2<>((String)auditLog.get("user"),auditLog));
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileMLAlgorithmEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileMLAlgorithmEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileMLAlgorithmEvaluator.java
deleted file mode 100644
index 42021ab..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/UserProfileMLAlgorithmEvaluator.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.core.ValuesArray;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.ml.MLAlgorithmEvaluator;
-import org.apache.eagle.ml.MLAnomalyCallback;
-import org.apache.eagle.ml.MLModelDAO;
-import org.apache.eagle.ml.impl.MLModelDAOImpl;
-import org.apache.eagle.ml.model.MLAlgorithm;
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.policy.PolicyEvaluationContext;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.apache.eagle.security.userprofile.model.UserProfileModel;
-import org.apache.eagle.service.client.EagleServiceConnector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public abstract class UserProfileMLAlgorithmEvaluator<M extends UserProfileModel> implements MLAlgorithmEvaluator,UserProfileAnomalyDetector<M> {
- protected transient MLAlgorithm algorithm;
- protected transient List<MLAnomalyCallback> callbacks;
- private transient Config config;
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileMLAlgorithmEvaluator.class);
- private transient MLModelDAO mlDAO;
- private transient UserProfileAnomalyDetector<M> detector;
- private transient PolicyEvaluationContext evalContext;
-
- @Override
- public void init(MLAlgorithm algorithm, Config config, PolicyEvaluationContext context) {
- this.algorithm = algorithm;
- this.callbacks = new ArrayList<>();
- this.config = config;
- this.mlDAO = this.getModelDAO();
- this.detector = getAnomalyDetector();
- }
-
- protected UserProfileAnomalyDetector<M> getAnomalyDetector(){ return this;}
-
- protected MLModelDAO getModelDAO(){
- // TODO: 1. Adapt to different model protocol including service, hdfs, file or kafka
- // TODO: 2. Model cache configuration
- return new MLModelDAOImpl(new EagleServiceConnector(this.config));
- }
-
- /**
- * @param data ValuesArray[0:alertContext{alertExecutor,policyId,evaluator,outputCollector},1:streamName,2:entity]
- *
- * @throws Exception
- */
- @Override
- public void evaluate(ValuesArray data) throws Exception {
- Collector collector = (Collector)data.get(0);
- evalContext.outputCollector = collector;
- String streamName = (String)data.get(1);
-
- UserActivityAggModelEntity userActivityAggModelEntity = (UserActivityAggModelEntity) data.get(2);
-
- String user = userActivityAggModelEntity.getTags().get(UserProfileConstants.USER_TAG);
- String algorithm = this.algorithm.getName();
-
- if(algorithm == null) LOG.warn("Algorithm name is null: "+this.algorithm);
-
- List<MLModelAPIEntity> mlModels = this.mlDAO.findMLModelByContext(user, algorithm);
- M model;
- if(mlModels == null || mlModels.size() == 0){
- LOG.warn(String.format("No persisted ML model found for user [%s], algorithm [%s]",user,algorithm));
- }else{
- if(mlModels.size()>0){
- LOG.warn(String.format("%s models found for user [%s], algorithm [%s]",mlModels.size(),user,algorithm));
- }
- for (MLModelAPIEntity mlModel : mlModels) {
- model = detector.convert(mlModel);
- //LOG.info("executing algorithm: " + );
- List<MLCallbackResult> callbackResults = this.detector.detect(user, algorithm, UserActivityAggModelEntity.toModel(userActivityAggModelEntity), model);
- if (callbackResults != null && callbackResults.size() >0) {
- notifyCallbacks(callbackResults, evalContext);
- } else {
- LOG.info(String.format("No anomaly activities detected for user [%s] by algorithm [%s] ",user,algorithm));
- }
- }
- }
- }
-
- protected void notifyCallbacks(List<MLCallbackResult> callbackResults, PolicyEvaluationContext alertContext){
- for(MLCallbackResult callbackResult: callbackResults){
- if(callbackResult.isAnomaly()) {
- for (MLAnomalyCallback callback : this.callbacks) {
- callback.receive(callbackResult,alertContext);
- }
- }
- }
- }
-
- @Override
- public void register(MLAnomalyCallback callback) throws Exception {
- this.callbacks.add(callback);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileEigenEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileEigenEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileEigenEvaluator.java
deleted file mode 100644
index 4c1a0e3..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileEigenEvaluator.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.security.userprofile.UserProfileMLAlgorithmEvaluator;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.eagle.security.userprofile.UserProfileMLAlgorithmEvaluator;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModelEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.apache.eagle.security.userprofile.model.UserProfileEigenModelEntity.deserializeModel;
-
-public abstract class AbstractUserProfileEigenEvaluator extends UserProfileMLAlgorithmEvaluator<UserProfileEigenModel> {
- private final static Logger LOG = LoggerFactory.getLogger(AbstractUserProfileEigenEvaluator.class);
- @Override
- public UserProfileEigenModel convert(MLModelAPIEntity entity) throws IOException {
- try {
- UserProfileEigenModel model = UserProfileEigenModelEntity.deserializeModel(entity);
- if(model == null || model.uMatrix() == null || model.diagonalMatrix() == null || model.minVector() == null || model.maxVector() == null
- || model.maximumL2Norm() == null || model.minimumL2Norm() == null || model.dimension() == 0){
- if(LOG.isDebugEnabled()) LOG.debug("Eigen model seems to not have meaningful information, ignore");
- return null;
- }else {
- return model;
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(),e);
- throw e;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileKDEEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileKDEEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileKDEEvaluator.java
deleted file mode 100644
index 49afcf4..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/AbstractUserProfileKDEEvaluator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLModelAPIEntity;
-import org.apache.eagle.security.userprofile.UserProfileMLAlgorithmEvaluator;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModelEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public abstract class AbstractUserProfileKDEEvaluator extends UserProfileMLAlgorithmEvaluator<UserProfileKDEModel> {
- private final static Logger LOG = LoggerFactory.getLogger(AbstractUserProfileKDEEvaluator.class);
-
- @Override
- public UserProfileKDEModel convert(MLModelAPIEntity entity) throws IOException {
- try {
- return UserProfileKDEModelEntity.deserializeModel(entity);
- } catch (IOException e) {
- LOG.error(e.getMessage(),e);
- throw e;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/TimeWindowImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/TimeWindowImpl.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/TimeWindowImpl.java
deleted file mode 100644
index 2326013..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/TimeWindowImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.security.userprofile.TimeWindow;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * @since 9/29/15
- */
-public class TimeWindowImpl implements TimeWindow{
- private final Long startTimestamp;
- private final Long endTimestamp;
- private final AtomicBoolean expired;
- private final Long safeWindowMs;
-
- public TimeWindowImpl(Long startTimestamp, Long endTimestamp, Long safeWindowMs) {
- this.startTimestamp = startTimestamp;
- this.endTimestamp = endTimestamp;
- this.expired = new AtomicBoolean(false);
- this.safeWindowMs = safeWindowMs;
- }
-
- public TimeWindowImpl(Long startTimestamp, Long endTimestamp) {
- this.startTimestamp = startTimestamp;
- this.endTimestamp = endTimestamp;
- this.expired = new AtomicBoolean(false);
- this.safeWindowMs = 0l;
- }
-
- @Override
- public long from() {
- return this.startTimestamp;
- }
-
- @Override
- public long to() {
- return this.endTimestamp;
- }
-
- @Override
- public boolean accept(Long timestamp) {
- if(timestamp > endTimestamp + safeWindowMs){
- this.expired.set(true);
- return false;
- }else if(timestamp >= startTimestamp){
- return true;
- }
- return false;
- }
-
- @Override
- public boolean expire() {
- return expired.get();
- }
-
- @Override
- public int hashCode() {
- HashCodeBuilder builder = new HashCodeBuilder();
- builder.append(this.getClass().getCanonicalName());
- builder.append(startTimestamp);
- builder.append(endTimestamp);
- return builder.toHashCode();
- }
-
- @Override
- public String toString() {
- return String.format("[%s - %s]",DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.startTimestamp),DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.endTimestamp));
- }
-}
\ No newline at end of file
[5/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
deleted file mode 100644
index 52bdbf0..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.security.userprofile.TimeWindow;
-import org.apache.eagle.security.userprofile.UserActivityAggregator;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.UserProfileUtils;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.*;
-
-/**
- * UserActivityAggregator implementation
- *
- * @see org.apache.eagle.security.userprofile.UserActivityAggregator
- * @since 9/28/15
- */
-public class UserActivityAggregatorImpl implements UserActivityAggregator {
- private final Map<Long,TimeWindow> timeWindowCache = Collections.synchronizedMap(new HashMap<Long, TimeWindow>());
- private final Map<TimeWindow,Map<Tuple2<String,String>,Double>> counterCache = Collections.synchronizedMap(new HashMap<TimeWindow, Map<Tuple2<String, String>, Double>>());
- private final static Logger LOG = LoggerFactory.getLogger(UserActivityAggregatorImpl.class);
- private final String site;
- private final long safeWindowMs;
- private final Period granularity;
- private final List<String> cmdFeatures;
-
- public UserActivityAggregatorImpl(List<String> cmdFeatures,Period granularity,String site,long safeWindowMs){
- this.cmdFeatures = cmdFeatures;
- this.granularity = granularity;
- this.site = site;
- this.safeWindowMs = safeWindowMs;
- }
-
- /**
- * Build new time window
- *
- * @param from where time window start from
- * @param to where time window end at
- * @return new time window instance
- */
- private TimeWindow newTimeWindowInstance(Long from, Long to){
- return new TimeWindowImpl(from,to,this.safeWindowMs);
- }
-
- @Override
- public void accumulate(Map<String, Object> event, Collector<Tuple2<String, UserActivityAggModelEntity>> collector) {
- Long timestamp = (Long) event.get("timestamp");
- Long baseTimestamp = UserProfileUtils.formatMillisecondsByPeriod(timestamp, this.granularity);
-
- TimeWindow window = timeWindowCache.get(baseTimestamp);
- if(window == null) {
- window = newTimeWindowInstance(baseTimestamp, baseTimestamp + this.granularity.toStandardSeconds().getSeconds() * 1000);
- LOG.info(String.format("Creating %s",window.toString()));
- // Initialize time window
- init(window);
- }
- List<TimeWindow> expiredWindows = new ArrayList<>();
- for(TimeWindow win:timeWindowCache.values()){
- if(win.accept(timestamp)) accumulate(win, event);
- if(win.expire()){
- // Flush time window
- flush(win, collector);
- expiredWindows.add(win);
- }
- }
-
- // clean expired time window
- for(TimeWindow win:expiredWindows){
- if(LOG.isDebugEnabled()) LOG.debug(String.format("%s is expired at [%s]", win.toString(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(timestamp)));
- close(win);
- }
- }
-
- private void close(TimeWindow win) {
- counterCache.remove(win);
- timeWindowCache.remove(win.from());
- }
-
- private void init(TimeWindow window) {
- timeWindowCache.put(window.from(),window);
- Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
- if(counter == null){
- counter = new HashMap<>();
- counterCache.put(window,counter);
- }
- }
-
- private void accumulate(TimeWindow window, Map<String, Object> event) {
- Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
- String user = (String) event.get("user");
- String cmd = (String) event.get("cmd");
- Tuple2<String,String> tuple = new Tuple2<>(user,cmd);
- if(counter.containsKey(tuple)){
- Double count = counter.get(tuple) + 1;
- counter.put(tuple,count);
- }else{
- counter.put(tuple,1.0);
- }
- }
-
- private void flush(TimeWindow window, Collector<Tuple2<String, UserActivityAggModelEntity>> collector){
- Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
-
- Map<String,Map<String,Double>> tmp = new HashMap<>();
- for(Map.Entry<Tuple2<String,String>,Double> entry:counter.entrySet()){
- String user = entry.getKey()._1();
- Map<String,Double> cmdCount = tmp.get(user);
- if(cmdCount == null) cmdCount = new HashMap<>();
- cmdCount.put(entry.getKey()._2(),entry.getValue());
- tmp.put(user,cmdCount);
- }
-
- for(Map.Entry<String,Map<String,Double>> entry:tmp.entrySet()){
- final String user = entry.getKey();
- final Map<String,Double> cmdCountMap = entry.getValue();
-
- UserActivityAggModelEntity model = new UserActivityAggModelEntity();
- model.setTags(new HashMap<String, String>(){{
- put(UserProfileConstants.SITE_TAG,site);
- put(UserProfileConstants.USER_TAG,user);
- }});
-
- // Build matrix
- double[][] matrix = new double[1][this.cmdFeatures.size()];
-
- for(int i=0;i<cmdFeatures.size();i++){
- Double cnt = cmdCountMap.get(cmdFeatures.get(i));
- if(cnt == null) cnt = 0.0;
- matrix[0][i] = cnt;
- }
- model.setCmdTypes(cmdFeatures);
- model.setCmdMatrix(matrix);
- model.setTimestamp(window.to());
-
- if(LOG.isDebugEnabled()) {
- try {
- LOG.debug(new ObjectMapper().writeValueAsString(model));
- } catch (JsonProcessingException e) {
- LOG.error(e.getMessage(), e);
- }
- }
- collector.collect(new Tuple2(user, model));
- }
- LOG.info(String.format("Flushed %s records during %s", tmp.size(), window));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
deleted file mode 100644
index db7897f..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class UserProfileAnomalyEigenEvaluator extends AbstractUserProfileEigenEvaluator {
- private static Logger LOG = LoggerFactory.getLogger(UserProfileAnomalyEigenEvaluator.class);
- private RealMatrix normalizeData(RealMatrix matrix, UserProfileEigenModel model){
- RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
- if(LOG.isDebugEnabled()) LOG.debug("model statistics size: " + model.statistics().length);
- for(int i=0; i< matrix.getRowDimension(); i++){
- for(int j=0; j < matrix.getColumnDimension(); j++){
- double value = (matrix.getEntry(i, j) - model.statistics()[j].getMean())/model.statistics()[j].getStddev();
- normalizedData.setEntry(i, j, value);
- }
- }
- return normalizedData;
- }
-
- @Override
- public List<MLCallbackResult> detect(final String user, final String algorithm, UserActivityAggModel userActivity, UserProfileEigenModel aModel) {
- RealMatrix inputData = userActivity.matrix();
- LOG.warn("EigenBasedAnomalyDetection predictAnomaly called with dimension: " + inputData.getRowDimension() + "x" + inputData.getColumnDimension());
-
- if(aModel== null){
- LOG.warn("nothing to do as the input model does not have required values, returning from evaluating this algorithm..");
- return null;
- }
-
- List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>();
- RealMatrix normalizedMat = normalizeData(inputData, aModel);
-
- UserCommandStatistics[] listStats = aModel.statistics();
- int colWithHighVariant = 0;
-
- for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
- if (listStats[j].isLowVariant() == false) {
- colWithHighVariant++;
- }
- }
-
- final Map<String,String> context = new HashMap<String, String>(){{
- put(UserProfileConstants.USER_TAG,user);
- put(UserProfileConstants.ALGORITHM_TAG,algorithm);
- }};
-
- Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>();
- for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
- MLCallbackResult aResult = new MLCallbackResult();
- aResult.setAnomaly(false);
- aResult.setContext(context);
-
- for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
- //LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
- //LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
- if (listStats[j].isLowVariant() == true) {
- //LOG.info(listOfCmds[j] + " is low variant");
- if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) {
- lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly");
- aResult.setAnomaly(true);
- aResult.setTimestamp(userActivity.timestamp());
- aResult.setFeature(listStats[j].getCommandName());
- aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
- List<String> datapoints = new ArrayList<String>();
- double[] rowVals = inputData.getRow(i);
- for(double rowVal:rowVals)
- datapoints.add(rowVal+"");
- aResult.setDatapoints(datapoints);
- aResult.setId(user);
- //mlCallbackResults.add(aResult);
- } /*else {
- aResult.setAnomaly(false);
- aResult.setTimestamp(userActivity.timestamp());
- mlCallbackResults.add(aResult);
- }*/
- }
- }
- mlCallbackResults.add(i, aResult);
- //return results;
- }
-
- //LOG.info("results size here: " + results.length);
-
- //LOG.info("col with high variant: " + colWithHighVariant);
- RealMatrix finalMatWithoutLowVariantFeatures = new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant);
- //LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension() +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension());
- int finalMatrixRow = 0;
- int finalMatrixCol = 0;
- for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
- for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
- if (listStats[j].isLowVariant() == false) {
- finalMatWithoutLowVariantFeatures.setEntry(finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j));
- finalMatrixCol++;
- }
- }
- finalMatrixCol = 0;
- finalMatrixRow++;
- }
- RealVector[] pcs = aModel.principalComponents();
- //LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension());
-
- RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose();
-
- for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) {
- if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) {
- //MLCallbackResult result = new MLCallbackResult();
- MLCallbackResult result = mlCallbackResults.get(i);
- //result.setContext(context);
- for (int sz = 0; sz < pcs.length; sz++) {
- double[] pc1 = pcs[sz].toArray();
- RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1);
- RealMatrix transposePC1Mat = pc1Mat.transpose();
- RealMatrix testData = pc1Mat.multiply(transposePC1Mat).multiply(finalInputMatTranspose.getColumnMatrix(i));
- //LOG.info("testData size: " + testData.getRowDimension() + "x" + testData.getColumnDimension());
- RealMatrix testDataTranspose = testData.transpose();
- //LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" + testDataTranspose.getColumnDimension());
- RealVector iRowVector = testDataTranspose.getRowVector(0);
- //RealVector pc1Vector = transposePC1Mat.getRowVector(sz);
- RealVector pc1Vector = transposePC1Mat.getRowVector(0);
- double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector);
- //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz));
- //LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz));
- if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) {
- //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz));
- result.setAnomaly(true);
- result.setFeature(aModel.statistics()[sz].getCommandName());
- result.setTimestamp(System.currentTimeMillis());
- result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
- List<String> datapoints = new ArrayList<String>();
- double[] rowVals = inputData.getRow(i);
- for(double rowVal:rowVals)
- datapoints.add(rowVal+"");
- result.setDatapoints(datapoints);
- result.setId(user);
- }
- }
- mlCallbackResults.set(i, result);
- }
- }
- return mlCallbackResults;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
deleted file mode 100644
index 7e89b28..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.collection.JavaConversions;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class UserProfileAnomalyKDEEvaluator extends AbstractUserProfileKDEEvaluator{
- private static Logger LOG = LoggerFactory.getLogger(UserProfileAnomalyKDEEvaluator.class);
-
- @Override
- public List<MLCallbackResult> detect(final String user, final String algorithm, UserActivityAggModel userActivity, UserProfileKDEModel aModel) {
- List<MLCallbackResult> mlPredictionOutputList = new ArrayList<MLCallbackResult>();
- RealMatrix inputData = userActivity.matrix();
-
- double[] probabilityEstimation = new double[inputData.getRowDimension()];
- for(int i=0; i < probabilityEstimation.length; i++)
- probabilityEstimation[i] = 1.0;
-
- boolean[][] anomalyFeature = new boolean[inputData.getRowDimension()][inputData.getColumnDimension()];
-
- for(int i=0; i < anomalyFeature.length;i++){
- for(int j=0; j < anomalyFeature[i].length; j++){
- anomalyFeature[i][j] = false;
- }
- }
-
- if(aModel == null){
- LOG.info("No model available for this uer, returning");
- return null;
- }
-
- Map<String,String> context = new HashMap<String,String>(){{
- put(UserProfileConstants.USER_TAG,user);
- put(UserProfileConstants.ALGORITHM_TAG,algorithm);
- }};
-
- for(int i=0; i<inputData.getRowDimension();i++){
- List<String> cmds = JavaConversions.seqAsJavaList(userActivity.cmdTypes());
- if(inputData.getColumnDimension() != cmds.size()){
- LOG.error("Test data is not with same dimension as training, aborting...");
- return null;
- }else{
-
- UserCommandStatistics[] listStats = aModel.statistics();
-
- for(int j=0; j < inputData.getColumnDimension(); j++){
-// LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
-// LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
- if(listStats[j].isLowVariant()){
-// LOG.info(listStats[j].getCommandName() + " is low variant for user: " + user);
- if(inputData.getEntry(i, j) > listStats[j].getMean()){
- probabilityEstimation[i]*= Double.NEGATIVE_INFINITY;
- anomalyFeature[i][j] = true;
- }
- }else{
- double stddev = listStats[j].getStddev();
- //LOG.info("stddev: " + stddev);
- double mean = listStats[j].getMean();
- //LOG.info("mean: " + mean);
- double sqrt2PI = Math.sqrt(2.0*Math.PI);
- //LOG.info("sqrt2PI: " + sqrt2PI);
- double denominatorFirstPart = sqrt2PI*stddev;
- //LOG.info("denominatorFirstPart: " + denominatorFirstPart);
- double squareMeanNormal = Math.pow((inputData.getEntry(i, j) - mean), 2);
- //LOG.info("squareMeanNormal: " + squareMeanNormal);
- double twoPowStandardDev = Math.pow(stddev, 2);
- //LOG.info("twoPowStandardDev: " + twoPowStandardDev);
- double twoTimesTwoPowStandardDev = 2.0*twoPowStandardDev;
- //LOG.info("twoTimesTwoPowStandardDev: " + twoTimesTwoPowStandardDev);
-
- double tempVal = ((1.00/denominatorFirstPart)
- *(Math.exp(-(squareMeanNormal/twoTimesTwoPowStandardDev))));
- probabilityEstimation[i] *= tempVal;
- //LOG.info("probabilityEstimation: " + probabilityEstimation[i]);
- if((inputData.getEntry(i, j) - mean) > 2*stddev)
- anomalyFeature[i][j] = true;
- }
- }
-
- }
- }
-
- for(int i=0; i < probabilityEstimation.length;i++){
- MLCallbackResult callBackResult = new MLCallbackResult();
- callBackResult.setContext(context);
- //LOG.info("probability estimation for data @" + i + " is: " + probabilityEstimation[i]);
- if(probabilityEstimation[i] < aModel.maxProbabilityEstimate()){
- callBackResult.setAnomaly(true);
- for(int col = 0 ; col < anomalyFeature[i].length; col++){
- //LOG.info("feature anomaly? " + (featureVals[col] == true));
- if(anomalyFeature[i][col] == true){
- callBackResult.setFeature(aModel.statistics()[col].getCommandName());
- }
- }
- }else{
- callBackResult.setAnomaly(false);
- }
-
- callBackResult.setTimestamp(userActivity.timestamp());
- List<String> datapoints = new ArrayList<String>();
- double[] rowVals = userActivity.matrix().getRow(i);
- for(double rowVal:rowVals)
- datapoints.add(rowVal+"");
- callBackResult.setDatapoints(datapoints);
- callBackResult.setId(user);
- callBackResult.setAlgorithm(UserProfileConstants.KDE_ALGORITHM);
- mlPredictionOutputList.add(callBackResult);
- }
- return mlPredictionOutputList;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider b/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
deleted file mode 100644
index d7a2754..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.eagle.ml.impl.MLPolicyEvaluatorServiceProviderImpl
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf b/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
deleted file mode 100644
index 8f2a690..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
+++ /dev/null
@@ -1,63 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
- "deployInstanceIdentifier" : {
- "programId" : "mlProgramId"
- },
- "envContextConfig" : {
- "env" : "storm",
- "topologyName" : "userProfileDetectionTopology",
- "mode" : "local",
- "parallelismConfig" : {
-
- },
- "stormConfigFile":"userprofile.storm.yaml"
- },
- "dataSourceConfig": {
- "topic" : "hdfs_audit_agg",
- "zkConnection" : "localhost:2181",
- "zkConnectionTimeoutMS" : 15000,
- "consumerGroupId" : "EagleConsumer",
- "fetchSize" : 1048586,
- "deserializerClass" : "org.apache.eagle.security.userprofile.model.UserActivityAggModelDeserializer",
- "transactionZKServers" : "localhost",
- "transactionZKPort" : 2181,
- "transactionZKRoot" : "/brokers/topics",
- "transactionStateUpdateMS" : 2000
- },
- "eagleProps" : {
- "site":"sandbox",
- "application":"userProfile",
- "eagleService": {
- "host": "localhost",
- "port": "38080",
- "username": "admin",
- "password": "secret"
- }
- },
- "alertExecutorConfigs" : {
- "userProfileAnomalyDetectionExecutor" : {
- "parallelism" : 1,
- "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner",
- "needValidation":true
- }
- },
- "dynamicConfigSource" : {
- "enabled" : true,
- "initDelayMillis" : 0,
- "delayMillis" : 5000
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf b/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
deleted file mode 100644
index 09bec08..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
+++ /dev/null
@@ -1,65 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
- "deployInstanceIdentifier" : {
- "programId" : "mlProgramId"
- },
- "envContextConfig" : {
- "env" : "storm",
- "topologyName" : "userProfileDetectionTopology",
- "mode" : "local",
- "parallelismConfig" : {
-
- },
- "stormConfigFile":"userprofile.storm.yaml"
- },
- "dataSourceConfig": {
- "topic" : "sandbox_hdfs_audit_log",
- "zkConnection" : "localhost:2181",
- "zkConnectionTimeoutMS" : 15000,
- "consumerGroupId" : "eagle.userprofile.detection",
- "fetchSize" : 1048586,
- "deserializerClass" : "org.apache.eagle.security.auditlog.HdfsAuditLogKafkaDeserializer",
- "transactionZKServers" : "localhost",
- "transactionZKPort" : 2181,
- "transactionZKRoot" : "/brokers/topics",
- "transactionStateUpdateMS" : 2000
- },
- "eagleProps" : {
- "site":"sandbox",
- "application":"userProfile",
- "userProfileFeatures":"",
- "userProfileMode":"streaming",
- "eagleService": {
- "host": "localhost",
- "port": 38080,
- "username": "admin",
- "password": "secret"
- }
- },
- "alertExecutorConfigs" : {
- "userProfileAnomalyDetectionExecutor" : {
- "parallelism" : 1,
- "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner",
- "needValidation":true
- }
- },
- "dynamicConfigSource" : {
- "enabled" : true,
- "initDelayMillis" : 0,
- "delayMillis" : 30000
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties b/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
deleted file mode 100644
index 3499c46..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt b/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
deleted file mode 100644
index d217229..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "type":"MachineLearning",
- "fileTypes":[
- {
- "type":"CSV",
- "containsHeader":"true",
- "fileParserClass":"eagle.ml.CSVFileFormatParser"
- }
- ],
- "alertContext":{
- "site":"sandbox",
- "component":"testComponent",
- "description":"ML based user profile anomaly detection",
- "severity":"WARNING",
- "notificationByEmail":"true"
- },
- "algorithms":[
- {
- "name":"eagle.security.userprofile.util.EigenBasedAnomalyDetection",
- "description":"EigenBasedAnomalyDetection",
- "modelPath":"/models/",
- "featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck",
- "type":"CSV"
- },
- {
- "name":"eagle.security.userprofile.util.KDEBasedAnomalyDetection",
- "description":"DensityBasedAnomalyDetection",
- "modelPath":"/models/",
- "featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck",
- "type":"CSV"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json b/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
deleted file mode 100644
index 4e63ae1..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
+++ /dev/null
@@ -1,18 +0,0 @@
-[
- {
- "prefix": "alertdef",
- "tags":{
- "programId" : "hdfsAuditLogMonitoring",
- "alertExecutorId" : "hdfsAuditLogAlertExecutor",
- "policyId" : "insecureCopy",
- "policyType" : "siddhiCEPEngine"
- },
- "desc": "insecure copy policy",
- "policyDef": "{\"type\":\"siddhiCEPEngine\",\"expression\":\"from hdfsAuditLogEventStream[cmd=='open' and src=='/tmp/private']#window.time(2 sec) select cmd, dst, host, src, user, timestamp, allowed insert into outputStream ;\",\"alertContext\":{\"description\":\"insecure copy file\"}}",
- "dedupeDef" : "{\"alertDedupIntervalMin\":\"1\",\"emailDedupIntervalMin\":\"1\"}",
- "notificationDef" : "",
- "remediationDef" : "",
- "enabled" : "true"
- }
-]
-
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml b/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
deleted file mode 100644
index 274aa2c..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java b/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
deleted file mode 100644
index 30bfaa3..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.security.userprofile.impl.UserActivityAggregatorImpl;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.joda.time.Period;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import scala.Tuple2;
-
-import java.text.ParseException;
-import java.util.*;
-
-@Ignore
-public class TestUserActivityAggregator {
- private UserActivityAggregator aggregator_0;
- private UserActivityAggregator aggregator_1;
-
- @Before
- public void setUp(){
- aggregator_0 = new UserActivityAggregatorImpl(Arrays.asList(UserProfileConstants.DEFAULT_CMD_TYPES),Period.parse("PT1m"),"test",0);
- aggregator_1 = new UserActivityAggregatorImpl(Arrays.asList(UserProfileConstants.DEFAULT_CMD_TYPES),Period.parse("PT1m"), "test", 5000);
- }
-
- @Test
- public void testAggWithoutSafeWindow() throws ParseException {
- SimpleCollector collector = new SimpleCollector();
- // yyyy-MM-dd HH:mm:ss,SSS
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"),"user1","open"),collector);
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"),"user1","open"),collector);
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:02,000"),"user1","open"),collector);
-
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"),"user2","open"),collector);
-
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"),"user1","open"),collector);
- aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:02,000"),"user2","open"),collector);
-
- Assert.assertEquals(2, collector.getResult().size());
- Assert.assertEquals("user2",collector.getResult().get(0)._1());
- Assert.assertEquals("user1",collector.getResult().get(1)._1());
- }
-
- @Test
- public void testAggWithSafeWindow() throws ParseException {
- SimpleCollector collector = new SimpleCollector();
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"),"user1","open"),collector); // create window
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"),"user1","rename"),collector);
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"),"user1","delete"),collector);
-
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"),"user1","open"),collector);
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:58,000"),"user2","open"),collector); // inside safe-window
-
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:06,000"),"user2","open"),collector); //
-
- aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:06:02,000"),"user2","open"),collector); // outside safe-window
-
- Assert.assertEquals(4, collector.getResult().size());
- Assert.assertEquals("user2",collector.getResult().get(0)._1());
- Assert.assertEquals("user1",collector.getResult().get(1)._1());
- }
-
-
- private Map<String,Object> event(final long timestamp, final String user, final String cmd){
- return new HashMap<String,Object>(){{
- put("timestamp",timestamp);
- put("user",user);
- put("cmd",cmd);
- }};
- }
-
- private class SimpleCollector implements Collector<Tuple2<String, UserActivityAggModelEntity>>{
- private List<Tuple2<String, UserActivityAggModelEntity>> result = new ArrayList<>();
- @Override
- public void collect(Tuple2<String, UserActivityAggModelEntity> tuple) {
- result.add(tuple);
- }
-
- public List<Tuple2<String, UserActivityAggModelEntity>> getResult(){
- return result;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
deleted file mode 100644
index eddea94..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.collection.JavaConversions;
-import scala.collection.Seq;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class TestUserProfileAnomalyEigenEvaluator {
-
- private static Logger LOG = LoggerFactory.getLogger(TestUserProfileAnomalyEigenEvaluator.class);
-
- @Test
- public void testDetect(){
- UserProfileAnomalyEigenEvaluator eigenEvaluator = new UserProfileAnomalyEigenEvaluator();
- String[] testCmdType = {"getfileinfo", "open", "listStatus", "setTimes", "setPermission", "rename",
- "mkdirs", "create", "setReplication", "contentSummary", "delete", "setOwner",
- "fsck"};
- List<String> tmpCmdTypesAsList = new ArrayList<String>();
- tmpCmdTypesAsList = Arrays.asList(testCmdType);
- Seq<String> testCmd = JavaConversions.asScalaBuffer(tmpCmdTypesAsList);
- String testSite = "sandbox";
- long testTimestamp = 14054440;
- String testUser = "test_user";
- RealMatrix testMatrix = new Array2DRowRealMatrix(1,testCmdType.length);
- for(int i=0; i < testMatrix.getColumnDimension();i++)
- testMatrix.addToEntry(0, i, 3.0);
-
- UserActivityAggModel testAggModel = new UserActivityAggModel(testUser, testMatrix, testCmd,testSite, testTimestamp);
-
- Long testVersion = new Long(1);
- RealMatrix testUMat = new Array2DRowRealMatrix(testCmdType.length,testCmdType.length);
- RealMatrix testDiagonalMat = new Array2DRowRealMatrix(testCmdType.length, testCmdType.length);
-
- for(int i=0; i< testCmdType.length; i++){
- for(int j=0; j < testCmdType.length; j++){
- testUMat.addToEntry(i,j,1.0);
- testDiagonalMat.addToEntry(i,j,1.0);
- }
- }
-
- int dimension = testCmdType.length -1;
- double[] minVector = new double[testCmdType.length];
- double[] maxVector = new double[testCmdType.length];
- for(int i=0; i < minVector.length;i++) {
- minVector[i] = 1;
- maxVector[i] = 1;
- }
-
- RealVector testMinVec = new ArrayRealVector(minVector);
- RealVector testMaxVec = new ArrayRealVector(maxVector);
- RealVector[] testPCs = new ArrayRealVector[testCmdType.length];
-
- for(int i =0; i < testCmdType.length; i++) {
- testPCs[i] = new ArrayRealVector(testCmdType.length);
- for(int j = 0; j < testPCs[i].getDimension(); j++){
- testPCs[i].addToEntry(j, 1.0);
- }
- }
-
- RealVector testMaxL2Norm = new ArrayRealVector(maxVector);
- RealVector testMinL2Norm = new ArrayRealVector(minVector);
- UserCommandStatistics userCommandStatistics[] = new UserCommandStatistics[testCmdType.length];
-
- for(int i=0; i< testCmdType.length;i++){
- userCommandStatistics[i] = new UserCommandStatistics();
- userCommandStatistics[i].setCommandName(testCmdType[i]);
- userCommandStatistics[i].setLowVariant(false);
- userCommandStatistics[i].setMean(1.0);
- userCommandStatistics[i].setStddev(1.0);
- }
-
- UserProfileEigenModel testEigenModel = new UserProfileEigenModel(testVersion,testSite,testUser,testUMat,testDiagonalMat,dimension, testMinVec, testMaxVec, testPCs, testMaxL2Norm, testMinL2Norm, userCommandStatistics);
-
- List<MLCallbackResult> testResults = eigenEvaluator.detect("test_user", "eigen", testAggModel, testEigenModel);
-
- Assert.assertEquals(testResults.size(), testMatrix.getRowDimension());
- for(MLCallbackResult result: testResults){
- Assert.assertEquals(result.isAnomaly(), true);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider b/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
deleted file mode 100644
index e363e67..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# org.apache.eagle.ml.legacy.MachineLearningPolicyEvaluatorServiceProviderImpl
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt b/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
deleted file mode 100644
index bb38d4a..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"type":"MachineLearning","fileTypes":[{"type":"CSV","containsHeader":"true","fileParserClass":"eagle.ml.CSVFileFormatParser"}],"alertContext":{"cluster":"testCluster","datacenter":"testDataCenter","component":"testComponent","description":"ML based user profile anomaly detection","severity":"WARNING","notificationByEmail":"true"},"algorithms":[{"name":"eagle.security.userprofile.legacy.legacy.util.EigenBasedAnomalyDetection","description":"EigenBasedAnomalyDetection","modelPath":"/models/","featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck","type":"CSV"},{"name":"eagle.security.userprofile.legacy.util.KDEBasedAnomalyDetection","description":"DensityBasedAnomalyDetection","modelPath":"/models/","featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck","type":"CSV"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1 b/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
deleted file mode 100644
index 2ea7262..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/pom.xml b/eagle-security/eagle-security-userprofile/pom.xml
deleted file mode 100644
index 0c112d3..0000000
--- a/eagle-security/eagle-security-userprofile/pom.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>eagle-security-parent</artifactId>
- <groupId>org.apache.eagle</groupId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>eagle-security-userprofile-parent</artifactId>
- <packaging>pom</packaging>
-
- <modules>
- <module>common</module>
- <module>detection</module>
- <module>training</module>
- </modules>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/README.md b/eagle-security/eagle-security-userprofile/training/README.md
deleted file mode 100644
index 79a42e4..0000000
--- a/eagle-security/eagle-security-userprofile/training/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-User Profiling Offline Training
-===============================
-
-Quick Start
------------
-1. Compile and build package
-
- mvn clean compile package -DskipTests
-
-2. Run with following command or directly on IDE
-
- spark-submit --class eagle.security.auditlog.AuditlogTrainingMain --master local target/eagle-security-mltraining-spark-0.1.0.jar
-
-Problems
---------
-1. **Problem**: 2015-07-19 17:18:42,083 ERROR [main] spark.SparkContext (Logging.scala:logError(96)) - Error initializing SparkContext. java.net.BindException: Failed to bind to: /10.249.64.134:0: Service 'sparkDriver' failed after 16 retries!
- **Solution**: Add environment variables
-
- export SPARK_LOCAL_IP=127.0.0.1
- export SPARK_MASTER_IP=127.0.0.1
-
-2. **Problem**: Detected both slf4j-log4j12 and log4j-over-slf4j in classpath
-
- **Solution**: Exclude log4 and slf4j related dependencies
-
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- </exclusion>
- </exclusions>
-
-3. **Problem**: fasterxml related class method not found
-
- **Solution**: Exclude following modules in version `2.4.1`:
-
- "com.fasterxml.jackson.core:jackson-core"
- "com.fasterxml.jackson.core:jackson-databind"
- "com.fasterxml.jackson.module:jackson-module-scala_2.10"
-
- but explicitly use `2.3.1` instead
-
-4. **Problem**: \u201corg.apache.commons.math3.exception.MathIllegalArgumentException: insufficient data: only 10 rows and 1 columns.\u201d breaks spark on distributed environment, but works locally.
-
- **Solution**: It's because Spark natively depends on math3 version "3.1.1".
-
-5. **Problem**: Exception in thread "delete Spark temp dirs" java.lang.NoClassDefFoundError: Could not initialize class org.apache.log4j.LogManager
-
- **Solution**: TODO
-
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
deleted file mode 100755
index 1b7907b..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-which spark-submit 1>/dev/null 2>&1
-
-if [ $? != 0 ];then
- echo "Error: spark-submit not found, please add spark-summit into environment PATH, exit code: 1"
- exit 1
-fi
-
-cd $(dirname $0)/../
-
-export JAR_FILE=$(ls `pwd`/target/eagle-security-userprofile-training-*-assembly.jar)
-export DRIVER_CLASSPATH=$(ls `pwd`/target/lib/asm-3.1.jar):$(ls `pwd`/target/lib/commons-math3-3.5.jar)
-
-#if [ ! -e $JAR_FILE ]; then
-# echo "$JAR_FILE not exists, build package ..."
-# mvn clean package -DskipTests
-# JAR_FILE=$(ls `pwd`/target/eagle-security-userprofile-training-*-assembly.jar)
-#else
-# echo "$JAR_FILE already exists"
-#
-#fi
-
-# Force clean and re-package
-mvn clean package -DskipTests
-
-export AUDIT_LOG_PATH=$(ls `pwd`/src/main/resources/hdfs-audit.log)
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.daemon.Scheduler" \
- -Dexec.args="-D eagle.userprofile.driver-classpath=$DRIVER_CLASSPATH -D eagle.userprofile.jar=$JAR_FILE -D eagle.userprofile.training-audit-path=$AUDIT_LOG_PATH -D eagle.userprofile.detection-audit-path=$AUDIT_LOG_PATH $@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
deleted file mode 100755
index f6da96a..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="$@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
deleted file mode 100755
index c4e53e1..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-export OUTPUT_PATH=/tmp/eagle-userprofile-output
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --kafka-props topic=hdfs_audit_agg,metadata.broker.list=sandbox.hortonworks.com:6667"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
deleted file mode 100755
index bd166d6..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-export OUTPUT_PATH=/tmp/eagle-userprofile-output
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-echo ""
-echo "CMD: mvn exec:java -DskipTests -Dexec.mainClass=\"eagle.security.userprofile.UserProfileTrainingCLI\" -Dexec.args=\"--input $HDFS_AUDIT\""
-echo ""
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --output $OUTPUT_PATH"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
deleted file mode 100755
index c35f3d1..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-echo ""
-echo "Execute: mvn exec:java -DskipTests -Dexec.mainClass=\"eagle.security.userprofile.UserProfileTrainingCLI\" -Dexec.args=\"--input $HDFS_AUDIT\""
-echo ""
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --service-host localhost --service-port 9098 --service-username admin --service-password secret"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
deleted file mode 100755
index e263928..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-which spark-submit 1>/dev/null 2>&1
-
-if [ $? != 0 ];then
- echo "Error: spark-submit not found, please add spark-summit into environment PATH, exit code: 1"
- exit 1
-fi
-
-cd $(dirname $0)/../
-
-cmd="mvn clean compile package -DskipTests"
-echo -e "Execute: $cmd\n"
-$cmd
-
-if [ $? != 0 ];then
- echo "-----"
- echo "Failed to compile and package project"
- exit 1
-else
- echo "Success to compile and package project"
-fi
-
-rm -rf /tmp/eagle-userprofile-output 1>/dev/null 2>&1
-cmd="spark-submit --class eagle.security.userprofile.UserProfileTrainingCLI `ls target/eagle-security-userprofile-training-*-assembly.jar` --master local[10] --site sandbox --input `pwd`/src/main/resources/hdfs-audit.log --output /tmp/eagle-userprofile-output"
-echo -e "Execute: $cmd\n"
-$cmd
-
-exit_code=$?
-if [ $exit_code != 0 ];then
- echo "-----"
- echo "Failed to execute job on spark, exit code: $exit_code"
- exit 1
-else
- echo "-----"
- echo "Success to execute job on spark, output: /tmp/eagle-userprofile-output, exit code: $exit_code"
-fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
deleted file mode 100755
index 3907a96..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SERVICE_HOST="localhost"
-export SERVICE_PORT=9098
-
-curl -H "Content-Type:application/json" -H "X-HTTP-Method-Override: POST" -XPOST "${SERVICE_HOST}:${SERVICE_PORT}/eagle-service/rest/entities?serviceName=ScheduleTaskService" -d "[{\"timestamp\":1441554892716,\"tags\": {\"site\":\"sandbox\",\"type\":\"USER_PROFILE_TRAINING\"},\"status\":\"INITIALIZED\",\"detail\":\"Newly created command\"}]" 2>/dev/null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/pom.xml b/eagle-security/eagle-security-userprofile/training/pom.xml
deleted file mode 100644
index 430cf95..0000000
--- a/eagle-security/eagle-security-userprofile/training/pom.xml
+++ /dev/null
@@ -1,219 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>eagle-security-userprofile-parent</artifactId>
- <groupId>org.apache.eagle</groupId>
- <version>0.5.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>eagle-security-userprofile-training</artifactId>
-
- <dependencies>
- <!-- spark needs asm 3, so we should use extcos 0.3b -->
- <dependency>
- <groupId>net.sf.extcos</groupId>
- <artifactId>extcos</artifactId>
- <version>${extcos3.version}</version>
- </dependency>
-
- <!-- Eagle -->
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>org.mortbay.jetty</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.eagle</groupId>
- <artifactId>eagle-security-userprofile-common</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!-- unit test -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </dependency>
-
- <!-- spark -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.scopt</groupId>
- <artifactId>scopt_${scala.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_${scala.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.version}</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
-
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>TestSuite.txt</filereports>
- </configuration>
- <executions>
- <execution>
- <id>test</id>
- <goals>
- <goal>test</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/lib</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <useFile>false</useFile>
- <disableXmlReport>true</disableXmlReport>
- <!-- If you have classpath issue like NoDefClassError,... -->
- <!-- useManifestOnlyJar>false</useManifestOnlyJar -->
- <includes>
- <include>**/*Test.*</include>
- <include>**/Test*.*</include>
- <include>**/*Suite.*</include>
- </includes>
- </configuration>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptor>src/assembly/assembly.xml</descriptor>
- <archive>
- <manifest>
- <mainClass>org.apache.eagle.security.userprofile.UserProfileTrainingCLI</mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <tarLongFileMode>posix</tarLongFileMode>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
[4/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml b/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
deleted file mode 100644
index 25d079c..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>assembly</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>.</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <unpack>true</unpack>
- <scope>runtime</scope>
- <unpackOptions>
- <excludes>
- <exclude>log4j.properties</exclude>
- </excludes>
- </unpackOptions>
- <excludes>
- <!--<exclude>org.apache.spark:*</exclude>-->
- <exclude>org.apache.storm:*</exclude>
- <exclude>org.clojure:*</exclude>
- <exclude>org.scala-lang:*</exclude>
- <exclude>org.apache.commons:math3</exclude>
- <exclude>com.fasterxml.jackson.core:*</exclude>
- <exclude>com.fasterxml.jackson.module:*</exclude>
- <exclude>org.slf4j:slf4j-api:*</exclude>
- <exclude>org.slf4j:slf4j-log4j12:*</exclude>
- <exclude>log4j:log4j:*</exclude>
- <exclude>org.slf4j:log4j-over-slf4j:*</exclude>
- <exclude>org.apache.hadoop:hadoop-yarn-*</exclude>
- <exclude>org.apache.hadoop:hadoop-mapreduce-*</exclude>
- <!--<exclude>org.apache.hbase:*</exclude>-->
- </excludes>
- </dependencySet>
- </dependencySets>
- <fileSets>
- <fileSet>
- <directory>${project.build.outputDirectory}</directory>
- <outputDirectory>.</outputDirectory>
- <excludes>
- <exclude>log4j.properties</exclude>
- <exclude>application.conf</exclude>
- <exclude>hdfs-audit.log</exclude>
- </excludes>
- </fileSet>
- </fileSets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
deleted file mode 100644
index 909b7b7..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.commons.math3.linear.RealMatrix;
-import scala.collection.JavaConversions;
-import scala.collection.immutable.List;
-
-/**
- * @since 10/21/15
- */
-public abstract class JavaUserProfileModeler<M,C extends UserProfileContext> implements UserProfileModeler<M,C> {
- @Override
- public List<M> build(String site, String user, RealMatrix matrix) {
- java.util.List<M> models = generate(site, user, matrix);
- if (models != null) {
- return JavaConversions.asScalaBuffer(models).toList();
- } else {
- return null;
- }
- }
-
- /**
- * Java delegate method for {@code eagle.security.userprofile.model.JavaUserProfileModeler#build(String site, String user, RealMatrix matrix)}
- *
- * @param site eagle site
- * @param user eagle user
- * @param matrix user activity matrix
- * @return Generate user profile model
- */
- public abstract java.util.List<M> generate(String site, String user, RealMatrix matrix);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
deleted file mode 100644
index cb9d660..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.eigen;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.exception.NotStrictlyPositiveException;
-import org.apache.commons.math3.linear.*;
-import org.apache.commons.math3.stat.correlation.Covariance;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class UserProfileEigenModeler extends JavaUserProfileModeler<UserProfileEigenModel,UserProfileContext> {
-
- private final String[] cmdTypes;
- private UserCommandStatistics[] statistics;
- private String outputLocation;
- private RealMatrix finalMatrixWithoutLowVariantCmds;
- private RealMatrix covarianceMatrix;
- private RealMatrix uMatrix;
- private RealMatrix vMatrix;
- private RealMatrix diagonalMatrix;
- private int dimension =0;
- private static final double MAINTENED_VARIANCE = 0.99;
- private RealVector[] principalComponents;
- private RealVector maximumL2Norm; //Double.NEGATIVE_INFINITY;
- private RealVector minimumL2Norm; //= Double.POSITIVE_INFINITY;
- private RealVector minVector;
- private RealVector maxVector;
-
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileEigenModeler.class);
- private static final double lowVarianceVal= 0.001;
-
- public RealVector[] getPrincipalComponents() {
- return principalComponents;
- }
-
- public void setPrincipalComponents(RealVector[] principalComponents) {
- this.principalComponents = principalComponents;
- }
-
- public UserProfileEigenModeler(String[] cmdTypes){
- this.cmdTypes = cmdTypes;
- }
-
- public UserProfileEigenModeler(){
- this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
- }
-
- public RealVector getMaximumL2Norm() {
- return maximumL2Norm;
- }
-
- public void setMaximumL2Norm(RealVector maximumL2Norm) {
- this.maximumL2Norm = maximumL2Norm;
- }
-
- public RealVector getMinimumL2Norm() {
- return minimumL2Norm;
- }
-
- public void setMinimumL2Norm(RealVector minimumL2Norm) {
- this.minimumL2Norm = minimumL2Norm;
- }
-
- public RealVector getMinVector() {
- return minVector;
- }
-
- public void setMinVector(RealVector minVector) {
- this.minVector = minVector;
- }
-
- public RealVector getMaxVector() {
- return maxVector;
- }
-
- public void setMaxVector(RealVector maxVector) {
- this.maxVector = maxVector;
- }
-
- public UserCommandStatistics[] getStatistics() {
- return statistics;
- }
-
- public void setStatistics(UserCommandStatistics[] statistics) {
- this.statistics = statistics;
- }
-
- public String getOutputLocation() {
- return outputLocation;
- }
-
- public void setOutputLocation(String outputLocation) {
- this.outputLocation = outputLocation;
- }
-
- private void computeStats(RealMatrix m){
-
- if(m.getColumnDimension() != this.cmdTypes.length){
- LOG.error("Please fix the commands list in config file");
- return;
- }
- statistics = new UserCommandStatistics[m.getColumnDimension()];
- for(int i=0; i<m.getColumnDimension(); i++){
- UserCommandStatistics stats = new UserCommandStatistics();
- stats.setCommandName(this.cmdTypes[i]);
- RealVector colData = m.getColumnVector(i);
- StandardDeviation deviation = new StandardDeviation();
- double stddev = deviation.evaluate(colData.toArray());
- //LOG.info("stddev is nan?" + (stddev == Double.NaN? "yes":"no"));
- if(stddev <= lowVarianceVal)
- stats.setLowVariant(true);
- else
- stats.setLowVariant(false);
- stats.setStddev(stddev);
- Mean mean = new Mean();
- double mu = mean.evaluate(colData.toArray());
- //LOG.info("mu is nan?" + (mu == Double.NaN? "yes":"no"));
- stats.setMean(mu);
- statistics[i] = stats;
- }
- }
-
- private RealMatrix normalizeData(RealMatrix matrix){
-
- RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
- for(int i=0; i< matrix.getRowDimension(); i++){
- for(int j=0; j < matrix.getColumnDimension(); j++){
- // TODO: statistics[j].getStddev() == 0 what should the value be if stddev is o?
- double value = statistics[j].getStddev() == 0? 0 : (matrix.getEntry(i, j) - statistics[j].getMean())/statistics[j].getStddev();
- normalizedData.setEntry(i, j, value);
- }
- }
- return normalizedData;
- }
-
- private void computeCovarianceAndSVD(RealMatrix inputMat, int containsLowVariantCol){
-
- int finalMatrixRow=0;
- int finalMatrixCol=0;
-
- LOG.info("containsLowVariantCol size: " + containsLowVariantCol);
- int colDimension = (inputMat.getColumnDimension() - containsLowVariantCol);
- try {
- finalMatrixWithoutLowVariantCmds = new Array2DRowRealMatrix(inputMat.getRowDimension(), colDimension);
- }catch (NotStrictlyPositiveException e){
- LOG.error(String.format("Failed to build matrix [rowDimension:%s, columnDimension: %s]",inputMat.getRowDimension(),colDimension),e);
- throw e;
- }
-
- for(int i=0; i < inputMat.getRowDimension(); i++){
- for(int j=0; j < inputMat.getColumnDimension();j++){
- if(!statistics[j].isLowVariant()){
- finalMatrixWithoutLowVariantCmds.setEntry(finalMatrixRow, finalMatrixCol, inputMat.getEntry(i, j));
- finalMatrixCol++;
- }
- }
- finalMatrixCol = 0;
- finalMatrixRow++;
- }
-
- Covariance cov;
- try {
- cov = new Covariance(finalMatrixWithoutLowVariantCmds.getData());
- }catch (Exception ex){
- throw new IllegalArgumentException(String.format("Failed to create covariance from matrix [ %s x %s ]",finalMatrixWithoutLowVariantCmds.getRowDimension(),finalMatrixWithoutLowVariantCmds.getColumnDimension()),ex);
- }
- covarianceMatrix = cov.getCovarianceMatrix();
- SingularValueDecomposition svd = new SingularValueDecomposition(covarianceMatrix);
- diagonalMatrix = svd.getS();
- uMatrix = svd.getU();
- vMatrix = svd.getV();
- }
-
- private void computeDimensionWithMaxVariance(){
- if(diagonalMatrix == null || diagonalMatrix.getRowDimension() != diagonalMatrix.getColumnDimension()){
- LOG.error("Diagonal matrix is not correctly computed, SVD should have been done");
- return;
- }
-
- double sumTotal = 0.0;
- for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
- sumTotal += diagonalMatrix.getEntry(i, i);
- }
- //LOG.info("sum total: " + sumTotal);
- double sumEach = 0.0;
- for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
- sumEach += diagonalMatrix.getEntry(i, i);
- if((sumEach/sumTotal) >= MAINTENED_VARIANCE){
- dimension = i;
- break;
- }
-
- }
- if(dimension > finalMatrixWithoutLowVariantCmds.getColumnDimension())
- LOG.error("Reduced dimension cannot be smaller than original size of the feature");
-
- }
-
- private void computePrincipalComponents(){
- principalComponents = new ArrayRealVector[uMatrix.getColumnDimension()];
- for(int i=0; i < uMatrix.getColumnDimension(); i++){
- principalComponents[i] = uMatrix.getColumnVector(i).mapMultiply(Math.sqrt(diagonalMatrix.getEntry(i, i)));
- }
- }
-
- private RealMatrix computeMaxDistanceOnPCs(int index){
-
- RealMatrix pc = new Array2DRowRealMatrix(principalComponents[index].toArray());
- RealMatrix transposePC1 = pc.transpose();
- RealMatrix finalDataTranspose = finalMatrixWithoutLowVariantCmds.transpose();
-
- RealMatrix trainingData = pc.multiply(transposePC1).multiply(finalDataTranspose);
- RealMatrix trainingDataTranspose = trainingData.transpose();
-
- double maxDistance = 0.0;
- double minDistance = 0.0;
- RealVector p1 = null, p2 = null;
- if(LOG.isDebugEnabled()) LOG.debug("Training data transpose size: " + trainingDataTranspose.getRowDimension() + "x" + trainingDataTranspose.getColumnDimension());
- for(int i = 0; i < trainingDataTranspose.getRowDimension(); i++){
- RealVector iRowVector = new ArrayRealVector(trainingDataTranspose.getRow(i));
- RealVector transposePC1Vect = transposePC1.getRowVector(0);
- double distance = iRowVector.getDistance(transposePC1Vect);
- if(distance > maxDistance){
- maxDistance = distance;
- p1 = iRowVector;
- p2 = transposePC1Vect;
- }
- if(distance < minDistance)
- minDistance = distance;
- //}
- }
- maximumL2Norm.setEntry(index, maxDistance);
- minimumL2Norm.setEntry(index, minDistance);
- minVector = p1;
- maxVector = p2;
- return trainingDataTranspose;
- }
-
- @Override
- public List<UserProfileEigenModel> generate(String site, String user, RealMatrix matrix) {
- LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
- computeStats(matrix);
- RealMatrix normalizedInputMatrix = normalizeData(matrix);
- int lowVariantColumnCount = 0;
- for(int j=0; j < normalizedInputMatrix.getColumnDimension();j++){
- if(statistics[j].isLowVariant()){
- lowVariantColumnCount++;
- }
- }
-
- if(normalizedInputMatrix.getColumnDimension() == lowVariantColumnCount){
- LOG.info("found user: " + user + " with all features being low variant. Nothing to do...");
- UserProfileEigenModel noopModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user,null,null,0,null,null,null,null,null,statistics);
- return Arrays.asList(noopModel);
- }
- else {
- computeCovarianceAndSVD(normalizedInputMatrix, lowVariantColumnCount);
- computeDimensionWithMaxVariance();
- computePrincipalComponents();
- maximumL2Norm = new ArrayRealVector(principalComponents.length);
- minimumL2Norm = new ArrayRealVector(principalComponents.length);
-
- for (int i = 0; i < principalComponents.length; i++) {
- RealMatrix trainingDataTranspose = computeMaxDistanceOnPCs(i);
- }
-
- UserProfileEigenModel userprofileEigenModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user, uMatrix, diagonalMatrix, dimension, minVector, maxVector, principalComponents, maximumL2Norm, minimumL2Norm, statistics);
- return Arrays.asList(userprofileEigenModel);
- }
- }
-
- @Override
- public UserProfileContext context() {
- return new UserProfileContext(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
deleted file mode 100644
index 1b24137..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.kde;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.JavaUserProfileModeler;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileContext;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class UserProfileKDEModeler extends JavaUserProfileModeler<UserProfileKDEModel,UserProfileContext> {
-
- private final String[] cmdTypes;
- private UserCommandStatistics[] statistics;
- private RealMatrix finalMatrixWithoutLowVariantCmds;
- private RealMatrix covarianceMatrix;
- private double[] probabilityEstimation;
- private double minProbabilityEstimate;
- private double maxProbabilityEstimate;
- private double nintyFivePercentileEstimate;
- private double medianProbabilityEstimate;
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileKDEModeler.class);
- private static final double lowVarianceVal= 0.001;
-
- public UserProfileKDEModeler(String[] cmdTypes){
- this.cmdTypes = cmdTypes;
- }
-
- public UserProfileKDEModeler(){
- this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
- }
-
- private void computeStats(RealMatrix m){
- if(m.getColumnDimension() != this.cmdTypes.length){
- LOG.error("Please fix the commands list in config file");
- }
-
- statistics = new UserCommandStatistics[m.getColumnDimension()];
-
- for(int i=0; i<m.getColumnDimension(); i++){
- UserCommandStatistics stats = new UserCommandStatistics();
- stats.setCommandName(this.cmdTypes[i]);
- RealVector colData = m.getColumnVector(i);
- StandardDeviation deviation = new StandardDeviation();
- double stddev = deviation.evaluate(colData.toArray());
-
- if(LOG.isDebugEnabled()) LOG.debug("Stddev is NAN ? " + (Double.isNaN(stddev) ? "yes" : "no"));
- if(stddev <= lowVarianceVal)
- stats.setLowVariant(true);
- else
- stats.setLowVariant(false);
-
- stats.setStddev(stddev);
- Mean mean = new Mean();
- double mu = mean.evaluate(colData.toArray());
- if(LOG.isDebugEnabled()) LOG.debug("mu is NAN ? " + (Double.isNaN(mu)? "yes":"no"));
-
- stats.setMean(mu);
- statistics[i]=stats;
- }
- }
-
- private void computeProbabilityDensityEstimation(RealMatrix inputMat){
-
- probabilityEstimation = new double[inputMat.getRowDimension()];
- for(int i=0; i < probabilityEstimation.length; i++)
- probabilityEstimation[i] = 1.0;
-
- for(int i=0; i < inputMat.getRowDimension(); i++){
- for(int j=0; j < inputMat.getColumnDimension(); j++){
- if(statistics[j].getStddev() > 0){
- double stddev = statistics[j].getStddev();
- double mean = statistics[j].getMean();
- double sqrt2PI = Math.sqrt(2.0*Math.PI);
- double denominatorFirstPart = sqrt2PI*stddev;
- double squareMeanNormal = Math.pow((inputMat.getEntry(i, j) - mean), 2);
- double twoPowStandardDev = Math.pow(stddev, 2);
- double twoTimesTwoPowStandardDev = 2.0*twoPowStandardDev;
- probabilityEstimation[i] *= ((1.00/denominatorFirstPart)
- *(Math.exp(-(squareMeanNormal/twoTimesTwoPowStandardDev))));
- }
- }
- }
-
- java.util.List<Double> listProb = new ArrayList<Double>();
- for(int i=0; i < probabilityEstimation.length; i++){
- probabilityEstimation[i] = Math.log10(probabilityEstimation[i]);
- listProb.add(probabilityEstimation[i]);
- }
-
- Collections.sort(listProb);
- int i=0;
- for(double d:listProb)
- probabilityEstimation[i++]=d;
-
- minProbabilityEstimate = probabilityEstimation[probabilityEstimation.length -1];
- maxProbabilityEstimate = probabilityEstimation[0];
-
- int len = probabilityEstimation.length;
- int nintyFivePercentIndex = (int)Math.round(0.05*len);
- int medianPercentIndex = (int)Math.round(0.5*len);
- if(medianPercentIndex >= len)
- medianProbabilityEstimate = probabilityEstimation[medianPercentIndex-1];
- else
- medianProbabilityEstimate = probabilityEstimation[medianPercentIndex];
- nintyFivePercentileEstimate = probabilityEstimation[nintyFivePercentIndex];
- }
-
- @Override
- public List<UserProfileKDEModel> generate(String site, String user, RealMatrix matrix) {
- LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
- computeStats(matrix);
- computeProbabilityDensityEstimation(matrix);
- UserProfileKDEModel userprofileKDEModel = new UserProfileKDEModel(System.currentTimeMillis(),site,user, statistics, minProbabilityEstimate, maxProbabilityEstimate, nintyFivePercentileEstimate, medianProbabilityEstimate);
- return Arrays.asList(userprofileKDEModel);
- }
-
- @Override
- public UserProfileContext context() {
- return new UserProfileContext(UserProfileConstants.KDE_ALGORITHM);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf b/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
deleted file mode 100644
index db896d0..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-eagle{
- site = "sandbox"
-
- service{
- host = "localhost"
- port = 9098
- username = "admin"
- password = "secret"
- }
-
- userprofile {
- jar="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/eagle-security-userprofile-training-0.1.0-assembly.jar"
- driver-shell="spark-submit"
- driver-classpath="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/lib/commons-math3-3.5.jar"
- // model-audit-path = "/tmp/audit/${yyyy-MM-*}/*.tar.gz"
- // agg-audit-path = "/tmp/audit/${yyyy-MM-dd}/${yyyy-MM-dd-hh}.tar.gz"
-
- training-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
- detection-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
- }
-}
\ No newline at end of file
[3/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log b/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log
deleted file mode 100644
index 93b8410..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log
+++ /dev/null
@@ -1,1920 +0,0 @@
-2015-07-09 00:00:00,172 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user dst=null perm=null proto=rpc
-2015-07-09 00:00:00,193 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/ambari-qa/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,196 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/ambari-qa/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,200 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/guest/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,205 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/guest/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,212 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hcat/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,215 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hcat/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,219 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hdfs/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,224 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=delete src=/user/hdfs/.Trash/150708180000 dst=null perm=null proto=rpc
-2015-07-09 00:00:00,227 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hdfs/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,230 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hive/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,234 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hive/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,237 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,242 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,251 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,254 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/oozie/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,259 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/root/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,262 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/root/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,265 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/solr/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,269 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/solr/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,274 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/spark/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,282 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/spark/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:00,285 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/yarn/.Trash dst=null perm=null proto=rpc
-2015-07-09 00:00:00,287 INFO FSNamesystem.audit: allowed=true ugi=hdfs (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/yarn/.Trash/Current dst=null perm=null proto=rpc
-2015-07-09 00:00:02,445 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:00:16,601 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:00:21,928 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:00:21,960 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:01:02,498 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:01:16,617 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:01:19,932 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:01:20,621 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:01:20,622 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:01:21,925 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:01:21,962 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:02:02,447 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:02:16,631 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:02:21,932 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:02:21,965 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:03:02,488 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:03:16,650 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:03:21,929 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:03:21,962 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:04:02,732 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:04:16,668 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:04:19,930 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:04:21,933 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:04:21,964 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:05:02,694 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:05:16,712 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:05:21,949 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:05:21,976 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:06:02,789 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:06:16,736 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:06:20,639 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:06:20,640 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:06:21,930 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:06:21,969 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:37:40,830 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data dst=null perm=null proto=rpc
-2015-07-09 00:37:40,833 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data dst=null perm=null proto=rpc
-2015-07-09 00:37:40,834 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/hbase.version dst=null perm=null proto=rpc
-2015-07-09 00:37:40,837 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=open src=/apps/hbase/data/hbase.version dst=null perm=null proto=rpc
-2015-07-09 00:37:40,845 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/hbase.id dst=null perm=null proto=rpc
-2015-07-09 00:37:40,847 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/hbase.id dst=null perm=null proto=rpc
-2015-07-09 00:37:40,853 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/hbase.id dst=null perm=null proto=rpc
-2015-07-09 00:37:40,855 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=open src=/apps/hbase/data/hbase.id dst=null perm=null proto=rpc
-2015-07-09 00:37:40,860 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/data/hbase/meta/1588230740 dst=null perm=null proto=rpc
-2015-07-09 00:37:40,861 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/data/hbase/meta/.tabledesc dst=null perm=null proto=rpc
-2015-07-09 00:37:40,864 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/data/hbase/meta/.tabledesc dst=null perm=null proto=rpc
-2015-07-09 00:37:40,867 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/data/hbase/meta/.tabledesc/.tableinfo.0000000007 dst=null perm=null proto=rpc
-2015-07-09 00:37:40,871 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=open src=/apps/hbase/data/data/hbase/meta/.tabledesc/.tableinfo.0000000007 dst=null perm=null proto=rpc
-2015-07-09 00:37:40,880 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/data/hbase/meta/.tmp/.tableinfo.0000000008 dst=null perm=null proto=rpc
-2015-07-09 00:37:40,887 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=create src=/apps/hbase/data/data/hbase/meta/.tmp/.tableinfo.0000000008 dst=null perm=hbase:hdfs:rw-r--r-- proto=rpc
-2015-07-09 00:37:40,910 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=mkdirs src=/apps/hbase/data/data/hbase/meta/.tabledesc dst=null perm=hbase:hdfs:rwxr-xr-x proto=rpc
-2015-07-09 00:37:40,913 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=rename src=/apps/hbase/data/data/hbase/meta/.tmp/.tableinfo.0000000008 dst=/apps/hbase/data/data/hbase/meta/.tabledesc/.tableinfo.0000000008 perm=hbase:hdfs:rw-r--r-- proto=rpc
-2015-07-09 00:37:40,914 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/data/hbase/meta/.tabledesc dst=null perm=null proto=rpc
-2015-07-09 00:37:40,917 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=delete src=/apps/hbase/data/data/hbase/meta/.tabledesc/.tableinfo.0000000007 dst=null perm=null proto=rpc
-2015-07-09 00:37:40,919 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.tmp dst=null perm=null proto=rpc
-2015-07-09 00:37:40,920 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/.tmp/data dst=null perm=null proto=rpc
-2015-07-09 00:37:40,922 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=delete src=/apps/hbase/data/.tmp dst=null perm=null proto=rpc
-2015-07-09 00:37:40,925 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=mkdirs src=/apps/hbase/data/.tmp dst=null perm=hbase:hdfs:rwxr-xr-x proto=rpc
-2015-07-09 00:37:40,926 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:37:40,941 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.snapshot dst=null perm=null proto=rpc
-2015-07-09 00:37:40,948 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot/.tmp dst=null perm=null proto=rpc
-2015-07-09 00:37:45,481 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/WALs dst=null perm=null proto=rpc
-2015-07-09 00:37:45,483 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/WALs dst=null perm=null proto=rpc
-2015-07-09 00:37:45,494 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/WALs/sandbox.hortonworks.com,60020,1436373302966 dst=null perm=null proto=rpc
-2015-07-09 00:37:45,497 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=rename src=/apps/hbase/data/WALs/sandbox.hortonworks.com,60020,1436373302966 dst=/apps/hbase/data/WALs/sandbox.hortonworks.com,60020,1436373302966-splitting perm=hbase:hdfs:rwxr-xr-x proto=rpc
-2015-07-09 00:37:45,498 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/WALs/sandbox.hortonworks.com,60020,1436373302966-splitting dst=null perm=null proto=rpc
-2015-07-09 00:37:45,500 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/WALs/sandbox.hortonworks.com,60020,1436373302966-splitting dst=null perm=null proto=rpc
-2015-07-09 00:38:01,890 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:38:08,379 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:38:11,553 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:38:13,554 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:38:13,593 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:39:02,445 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:39:08,387 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:39:13,558 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:39:13,594 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:40:02,570 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:40:08,401 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:40:13,556 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:40:13,595 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:41:02,362 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:41:08,417 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:41:11,554 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:41:13,563 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:41:13,598 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:42:02,385 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:42:08,429 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:42:12,240 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,242 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,245 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,245 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,246 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,247 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,249 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,250 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,251 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,252 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,254 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,255 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,257 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:12,258 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/apps/hbase/data/.hbase-snapshot dst=null perm=null proto=rpc
-2015-07-09 00:42:13,555 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/oldWALs dst=null perm=null proto=rpc
-2015-07-09 00:42:13,597 INFO FSNamesystem.audit: allowed=true ugi=hbase (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/apps/hbase/data/archive dst=null perm=null proto=rpc
-2015-07-09 00:43:02,363 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:43:08,438 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:44:02,317 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:44:08,448 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:44:11,554 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:45:02,399 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:45:08,462 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:46:02,247 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:46:08,475 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:47:02,500 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:47:08,493 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:47:11,555 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:48:02,309 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:48:08,504 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:49:02,324 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:49:08,517 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:50:02,437 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:50:08,526 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:50:11,557 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:51:02,422 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:51:08,540 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:52:02,207 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:52:08,556 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:53:02,238 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:53:08,571 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:53:11,554 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:54:02,256 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:54:08,585 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:55:02,169 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:55:08,603 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:56:02,242 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:56:08,618 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:56:11,553 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:57:02,363 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:57:08,636 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:58:02,280 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:58:08,653 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:59:02,344 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 00:59:08,671 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 00:59:11,556 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:00:02,287 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:00:08,688 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:01:02,157 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:01:08,702 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:02:02,198 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:02:08,719 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:02:11,554 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:03:02,207 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:03:08,734 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:04:02,373 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:04:08,747 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:05:02,379 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:05:08,765 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:05:11,554 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:06:02,271 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:06:08,785 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:07:02,262 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:07:08,805 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:08:02,166 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:08:08,822 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:08:11,555 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:09:02,334 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:09:08,838 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:10:02,118 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:10:08,858 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:10:45,417 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:45,423 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:45,426 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:45,436 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:45,444 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,867 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mr-history dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,875 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mr-history dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,884 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,892 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,898 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mr-history dst=null perm=null proto=webhdfs
-2015-07-09 01:10:46,904 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:48,070 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:48,076 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:48,083 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:48,090 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:48,097 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,369 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/demo dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,378 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/demo dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,386 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/demo dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,395 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,404 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/demo dst=null perm=null proto=webhdfs
-2015-07-09 01:10:49,414 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:50,246 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/demo/data dst=null perm=null proto=webhdfs
-2015-07-09 01:10:50,251 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/demo/data dst=null perm=null proto=webhdfs
-2015-07-09 01:10:50,257 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/demo/data dst=null perm=null proto=webhdfs
-2015-07-09 01:10:52,054 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:52,059 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:52,066 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:52,075 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:52,088 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,460 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,468 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,475 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,479 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,482 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:10:53,492 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,558 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/guest dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,563 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/guest dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,567 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/guest dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,572 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,577 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/guest dst=null perm=null proto=webhdfs
-2015-07-09 01:10:54,584 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:56,213 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:56,216 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:56,220 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:56,227 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:56,236 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,212 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mapred dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,217 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mapred dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,221 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mapred dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,226 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,230 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/mapred dst=null perm=null proto=webhdfs
-2015-07-09 01:10:57,234 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:10:59,131 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:59,137 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:59,142 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:59,149 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:10:59,164 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,252 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,258 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,266 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,274 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,279 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:00,284 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,613 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,617 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,621 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,631 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,637 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit dst=null perm=null proto=webhdfs
-2015-07-09 01:11:01,643 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:02,241 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hbaseMaster dst=null perm=null proto=webhdfs
-2015-07-09 01:11:02,256 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hbaseMaster dst=null perm=null proto=webhdfs
-2015-07-09 01:11:02,264 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/hbaseMaster dst=null perm=null proto=webhdfs
-2015-07-09 01:11:02,359 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:11:03,862 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hbaseRegional dst=null perm=null proto=webhdfs
-2015-07-09 01:11:03,867 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hbaseRegional dst=null perm=null proto=webhdfs
-2015-07-09 01:11:03,870 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/hbaseRegional dst=null perm=null proto=webhdfs
-2015-07-09 01:11:04,494 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hdfs dst=null perm=null proto=webhdfs
-2015-07-09 01:11:04,498 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hdfs dst=null perm=null proto=webhdfs
-2015-07-09 01:11:04,501 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/hdfs dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,027 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hiveServer2 dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,031 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/hiveServer2 dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,035 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/hiveServer2 dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,627 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/knox dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,634 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/knox dst=null perm=null proto=webhdfs
-2015-07-09 01:11:05,639 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/knox dst=null perm=null proto=webhdfs
-2015-07-09 01:11:06,154 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:06,160 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:06,169 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:08,003 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:08,014 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:08,021 INFO FSNamesystem.audit: allowed=false ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger/audit/storm dst=null perm=null proto=webhdfs
-2015-07-09 01:11:08,870 INFO FSNamesystem.audit: allowed=true ugi=oozie (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/oozie/share/lib dst=null perm=null proto=rpc
-2015-07-09 01:11:08,998 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,002 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,005 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,013 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,019 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ranger dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,026 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,699 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,703 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,707 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,713 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:09,720 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,012 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,018 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,023 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,027 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,031 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,039 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:11,555 INFO FSNamesystem.audit: allowed=true ugi=mapred (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/mr-history/tmp dst=null perm=null proto=rpc
-2015-07-09 01:11:22,133 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=delete src=/tmp/private dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,153 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,160 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,170 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,179 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,186 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,191 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,456 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,461 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,465 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,468 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/ dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,473 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/tmp dst=null perm=null proto=webhdfs
-2015-07-09 01:11:22,485 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,160 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,168 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,177 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,186 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,197 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,209 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,220 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,413 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,417 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,421 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,425 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,431 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,442 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:38,449 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,102 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,111 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,118 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,126 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,137 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,147 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,157 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,350 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,354 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,358 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,362 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,366 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,376 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,382 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,823 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,828 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,833 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,840 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,848 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,855 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:39,863 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,055 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,060 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,064 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,072 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,078 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,082 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,088 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,274 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,279 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,282 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,286 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,291 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,295 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,302 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,356 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,364 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,368 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,374 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,381 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,385 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,390 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,618 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,623 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,627 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,630 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,633 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,637 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,642 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,731 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,736 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,740 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,745 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,748 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,753 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,757 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,808 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,814 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,822 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,833 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,842 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,850 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:40,858 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,177 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,185 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,192 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,199 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,209 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,214 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,219 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,460 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,467 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,477 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,482 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,488 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,494 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,499 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,539 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,548 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,557 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,563 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,572 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,577 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,583 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,783 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,788 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,795 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,810 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,815 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,821 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,827 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,973 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,978 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,982 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,986 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,990 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:41,995 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,001 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,189 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,192 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,197 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,201 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,205 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,214 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,222 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,467 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,472 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,476 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,485 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,493 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,498 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,504 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,549 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,555 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,563 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,568 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,572 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,575 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,580 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,747 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,761 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,767 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,772 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,778 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,783 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,787 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,970 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,976 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,980 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,984 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,988 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,992 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:42,999 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,071 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,077 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,081 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,085 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,089 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,093 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,098 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,318 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,324 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,328 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,334 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,346 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,359 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,365 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,590 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,596 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,609 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,614 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,620 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,625 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,634 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,692 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,699 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,712 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,717 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,724 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,731 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,740 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,943 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,949 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,963 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,968 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=listStatus src=/user/hue dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,975 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (auth:SIMPLE) ip=/10.0.2.15 cmd=getfileinfo src=/user dst=null perm=null proto=webhdfs
-2015-07-09 01:11:43,979 INFO FSNamesystem.audit: allowed=true ugi=hue (auth:PROXY) via hue (aut
<TRUNCATED>
[2/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Posted by yo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/log4j.properties b/eagle-security/eagle-security-userprofile/training/src/main/resources/log4j.properties
deleted file mode 100644
index aff248b..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=DEBUG, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/reference.conf b/eagle-security/eagle-security-userprofile/training/src/main/resources/reference.conf
deleted file mode 100644
index c57f4a1..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/reference.conf
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-akka {
-
- # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs
- # to STDOUT)
- loggers = ["akka.event.slf4j.Slf4jLogger"]
-
- # Log level used by the configured loggers (see "loggers") as soon
- # as they have been started; before that, see "stdout-loglevel"
- # Options: OFF, ERROR, WARNING, INFO, DEBUG
- loglevel = "DEBUG"
-
- # Log level for the very basic logger activated during ActorSystem startup.
- # This logger prints the log messages to stdout (System.out).
- # Options: OFF, ERROR, WARNING, INFO, DEBUG
- stdout-loglevel = "DEBUG"
-
- # Filter of log events that is used by the LoggingAdapter before
- # publishing log events to the eventStream.
- logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/sbin/submit-userprofile-training.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/sbin/submit-userprofile-training.sh b/eagle-security/eagle-security-userprofile/training/src/main/sbin/submit-userprofile-training.sh
deleted file mode 100644
index 6d84357..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/sbin/submit-userprofile-training.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cd $(dirname $0)/../
-
-./bin/submit-userprofile-training.sh --master yarn-cluster \
- --input /logs/auditlog/* \
- --service-host localhost \
- --service-port 9099
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileJobFactory.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileJobFactory.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileJobFactory.scala
deleted file mode 100644
index 2bb8b5a..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileJobFactory.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile
-
-import org.apache.eagle.security.userprofile.job.AuditLogTrainingSparkJob
-import org.joda.time.Period
-
-/**
-* @since 7/21/15
-*/
-object UserProfileJobFactory {
- def AuditlogTrainingSparkJob(site:String=null,input:String=null, master:String = "local[1]", appName:String = UserProfileConstants.DEFAULT_TRAINING_APP_NAME, cmdTypes:Seq[String] = UserProfileConstants.DEFAULT_CMD_TYPES,period:Period)(f: AuditLogTrainingSparkJob => Unit = null) = {
- val app = new AuditLogTrainingSparkJob(site,input,master,appName,cmdTypes,period)
- if (f != null) f(app)
- app
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingApp.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingApp.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingApp.scala
deleted file mode 100644
index f704a61..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingApp.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile
-
-import java.util.Properties
-
-import org.apache.eagle.security.userprofile.model.eigen.UserProfileEigenModeler
-import org.apache.eagle.security.userprofile.model.kde.UserProfileKDEModeler
-import org.apache.eagle.security.userprofile.sink.{UserActivityAggKafkaSink, UserProfileHDFSSink, UserProfileEagleServiceSink}
-import org.apache.eagle.security.userprofile.UserProfileJobFactory._
-import org.apache.spark.Logging
-import org.joda.time.Period
-
-/**
- * @since 7/28/15
- */
-case class UserProfileTrainingApp(config:UserProfileTrainingConfig) extends Logging{
- AuditlogTrainingSparkJob(config.site,config.input, config.master, config.appName, config.cmdTypes, config.period) { job =>
- logInfo(s"Registering UserProfileEigenModeler(${job.cmdTypes.mkString(",")})")
- job.model(new UserProfileEigenModeler(job.cmdTypes.toArray))
- logInfo(s"Registering UserProfileKDEModeler(${job.cmdTypes.mkString(",")})")
- job.model(new UserProfileKDEModeler(job.cmdTypes.toArray))
-
- if (config.serviceHost != null) {
- logInfo(s"Registering UserProfileEagleServiceSink(${config.serviceHost},${config.servicePort}, ${config.username}, ${config.password})")
- job.sink(new UserProfileEagleServiceSink(config.serviceHost, config.servicePort, config.username, config.password))
- }
-
- if (config.modelOutput != null) {
- logInfo(s"Registering UserProfileHdfsSink(${config.modelOutput})")
- job.sink(new UserProfileHDFSSink(config.modelOutput))
- }
-
- if (config.kafkaProps != null) {
- logInfo(s"Registering UserActivityAggKafkaSink(${config.kafkaProps}))")
- job.sink(new UserActivityAggKafkaSink(config.kafkaProps))
- }
-
- logInfo("Starting to run")
-
- job.run()
- }
-}
-
-/**
- * @param input audit log input
- * @param period aggregation period
- * @param master spark master url
- * @param appName spark application name
- * @param cmdTypes command types
- * @param modelOutput model output path
- * @param serviceHost service host
- * @param servicePort service port
- */
-case class UserProfileTrainingConfig(site:String = null,
- input:String = null,
- period:Period = Period.parse("PT1M"), // changing it to 1M aggregation interval
- master:String="local[10]",
- appName:String="UserProfileTraining",
- cmdTypes:Seq[String]=UserProfileConstants.DEFAULT_CMD_TYPES,
- modelOutput:String=null,
- serviceHost:String=null,
- servicePort:Int = 9099,
- username:String=null,
- password:String=null,
- kafkaProps:Properties = null)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingCLI.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingCLI.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingCLI.scala
deleted file mode 100644
index feb6548..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/UserProfileTrainingCLI.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile
-
-import org.apache.commons.configuration.{MapConfiguration, ConfigurationConverter}
-import org.joda.time.Period
-
-import scala.collection.JavaConversions
-
-/**
- * Must be the same name with UserProfileTrainingCLI for class path loader
- *
- * @since 7/28/15
- */
-class UserProfileTrainingCLI{}
-
-object UserProfileTrainingCLI{
- val PARSER = new scopt.OptionParser[UserProfileTrainingConfig]("[submit-command] --class eagle.security.userprofile.UserProfileTrainingMain [jar]") {
- head("User Profile Training Application","0.0.1")
- opt[String]('s',"site") required() action {(x,c) => c.copy(site = x)} text "Site"
- opt[String]('i',"input") required() action {(x,c) => c.copy(input = x)} text "Input audit log file path"
- opt[String]('o',"output") optional() action {(x,c) => c.copy(modelOutput = x)} text "Model output HDFS directory, final output path format is: ${output}_${algorithm}"
- opt[String]('m',"master") optional() action {(x,c) => c.copy(master = x)} text "Spark master url, default: local[10]"
- opt[String]('n',"app-name") optional() action {(x,c) => c.copy(appName = x)} text "Application name, default: UserProfile"
- opt[Seq[String]]('c',"cmds") optional() action {(x,c) => c.copy(cmdTypes = x)} text s"Command types, default: [${UserProfileConstants.DEFAULT_CMD_TYPES.mkString(",")}]"
- opt[String]('h',"service-host") optional() action {(x,c) => c.copy(serviceHost = x)} text "Eagle service host, default: localhost"
- opt[Int]('p',"service-port") optional() action {(x,c) => c.copy(servicePort = x)} text "Eagle service port, default: 9099"
- opt[String]('u',"service-username") optional() action {(x,c) => c.copy(username = x)} text "Eagle service authentication username, default: admin"
- opt[String]('w',"service-password") optional() action {(x,c) => c.copy(password = x)} text "Eagle service authentication password, default: secure"
- opt[Map[String,String]]('k',"kafka-props") optional() action { (x,c) => {c.copy(kafkaProps = ConfigurationConverter.getProperties(new MapConfiguration(JavaConversions.mapAsJavaMap(x))))}} text "Kafka properties, for example: topic=sometopic,metadata.brokers.list=localhost:9200"
- opt[String]('r',"period") optional() action {(x,c) => c.copy(period = Period.parse(x))} text "Period window (https://en.wikipedia.org/wiki/ISO_8601#Durations), default: PT1M" // changing it to 1M interval
- }
-
- def main(args:Array[String]): Unit = {
- PARSER.parse(args,UserProfileTrainingConfig()) match {
- case Some(config) =>
- UserProfileTrainingApp(config)
- sys.exit(0)
- case None =>
- System.err.println("\nMore details: https://spark.apache.org/docs/latest/submitting-applications.html")
- sys.exit(1)
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Command.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Command.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Command.scala
deleted file mode 100644
index 3c9e5a0..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Command.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import org.apache.eagle.security.userprofile.daemon.SchedulerContext.{COMMAND_SOURCE, COMMAND_TYPE, _}
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity.STATUS
-
-import scala.util.{Failure, Success, Try}
-
-/**
- * @since 0.3.0
- */
-trait Executable[T]{
- def execute:(SchedulerContext => Try[T])
-}
-
-/**
- * @since 0.3.0
- */
-trait ShellExecutable extends Executable[String] {
- def shell:(SchedulerContext => Seq[String])
- override def execute = { context =>
- val command = shell(context)
- var result:Try[String] = null
- var process:Process = null
- try {
- val builder = new ProcessBuilder(scala.collection.JavaConversions.seqAsJavaList(command))
- builder.redirectOutput(ProcessBuilder.Redirect.INHERIT)
- builder.redirectError(ProcessBuilder.Redirect.INHERIT)
- process = builder.start()
- process.waitFor()
- val exitCode = process.exitValue()
- if(exitCode != 0 ) {
- result = Failure(new IllegalMonitorStateException(s"Exit code of process is not zero, but: $exitCode"))
- }else {
- result = Success(s"Successfully executed command $command")
- }
- }catch{
- case e: Exception => result = Failure(e)
- } finally {
- if(process!=null) {
- process.destroy()
- }
- }
- result
- }
-}
-
-abstract class Command(protected val timestamp:Long,protected val site:String,protected val category: COMMAND_TYPE.TYPE,source: COMMAND_SOURCE.TYPE,protected val status:ScheduleCommandEntity.STATUS, val updateTime:Long, val persistable:Boolean) {}
-
-abstract class UserProfileCommand(
- timestamp:Long,
- inputPath:String,
- site:String,
- override val category: COMMAND_TYPE.TYPE,
- status:STATUS,
- source: COMMAND_SOURCE.TYPE,
- updateTime:Long,
- persistable:Boolean ) extends Command(timestamp,site,category,source,status,updateTime,persistable) with ShellExecutable
-
-case class UserProfileModelCommand(override val timestamp:Long,
- inputPath:String,
- override val site:String,
- override val status:ScheduleCommandEntity.STATUS,
- source: COMMAND_SOURCE.TYPE,
- createTime:Long,
- override val updateTime:Long,
- override val persistable:Boolean = true ) extends UserProfileCommand(timestamp,inputPath,site,COMMAND_TYPE.USER_PROFILE_TRAINING,status,source,updateTime,persistable) {
-
- override def shell = { context: SchedulerContext =>
- var builder = Seq(context.driverShell)
- if(context.driverClasspath!=null)
- builder = builder ++ Seq("--driver-class-path",context.driverClasspath)
- builder ++ Seq (
- "--master",context.sparkMaster,
- "--deploy-mode",context.sparkMode,
- "--class",UserProfileTrainingCLIClass,context.jobJar,
- "--site",this.site,
- "--period",context.period,
- "--input", this.inputPath,
- "--service-host",context.eagleServiceContext.serviceHost,
- "--service-port",context.eagleServiceContext.servicePort.toString,
- "--service-username", context.eagleServiceContext.username,
- "--service-password", context.eagleServiceContext.password
- )
- }
-}
-
-object Command{
- def asEntity(command:Command):ScheduleCommandEntity = {
- val entity = new ScheduleCommandEntity
- val tags = new java.util.HashMap[String,String]
- tags.put("site",command.site)
- tags.put("type",command.category.toString)
- entity.setTags(tags)
- entity.setTimestamp(command.timestamp)
- entity.setUpdateTime(command.updateTime)
- entity.setStatus(command.status.toString)
- entity
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Consumers.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Consumers.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Consumers.scala
deleted file mode 100644
index 2a6ec5a..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Consumers.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import akka.actor._
-import org.apache.eagle.common.EagleExceptionWrapper
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity.STATUS
-
-import scala.util.{Failure, Success}
-
-/**
- * @since 9/10/15
- */
-abstract class CommandConsumer extends UntypedActor with ActorLogging{
- @volatile var dao:UserProfileCommandDao = null
- @volatile var config:SchedulerContext=null
-
- override def onReceive(message: Any): Unit = message match {
- case _config: SchedulerContext =>
- if(log.isDebugEnabled) {
- log.debug(s"Initialized with config: $config")
- } else {
- if(config == null) log.info(s"Initialized") else log.info("Reinitialized")
- }
- config = _config
- dao = new UserProfileCommandDao(_config.eagleServiceContext.serviceHost, _config.eagleServiceContext.servicePort, _config.eagleServiceContext.username, _config.eagleServiceContext.password, this)
- case _command: Command =>
- val _config = config
- if(_config == null) throw new IllegalStateException("Config is called before asigned")
- val _sender = sender()
- handle.apply(_command,_config,_sender)
- case _ => throw new IllegalArgumentException(s"Unknown message $message")
- }
- def handle: PartialFunction[(Command,SchedulerContext,ActorRef), Any]
-}
-
-class UserProfileCommandConsumer extends CommandConsumer{
- import context.dispatcher
- override def handle = {
- case (command, cfg, sender) => command match {
- case c:UserProfileCommand =>
- val entity = Command.asEntity(command)
- if(c.persistable) {
- dao.updateCommandStatus(entity, STATUS.EXECUTING, s"Command [$entity] is executing ") onComplete { case _ => execute(c,entity,cfg) }
- }else{
- execute(c,entity,cfg)
- }
- }
- }
-
- private def execute(cmd: UserProfileCommand, entity:ScheduleCommandEntity,cfg: SchedulerContext): Unit ={
- cmd.execute(cfg) match {
- case Success(message) =>
- log.info(s"Executed successfully, returned: $message")
- if (cmd.persistable)
- dao.updateCommandStatus(entity, STATUS.SUCCEEDED, message) recover {
- case ex: Throwable => log.error(ex, s"Failed to update status of [$entity] as SUCCEEDED, due to exception" + ex.getMessage)
- }
- case Failure(ex) =>
- log.error(ex, s"Failed to execute: ${cmd.shell},due to: ${ex.getMessage}")
- if (cmd.persistable) {
- dao.updateCommandStatus(entity, STATUS.FAILED, EagleExceptionWrapper.wrap(ex.asInstanceOf[Exception])) recover {
- case ex: Throwable => log.error(ex, s"Failed to update status of [$entity] as SUCCEEDED, due to exception: " + ex.getMessage)
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Coordinator.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Coordinator.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Coordinator.scala
deleted file mode 100644
index 4b952ea..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Coordinator.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import akka.actor.{Props, _}
-
-/**
- * @since 9/11/15
- */
-case class Initialized(context: SchedulerContext)
-case class CheckOndemandTrainingStatus(var site: String, var category: SchedulerContext.COMMAND_TYPE.TYPE)
-case class CheckPeriodicTrainingStatus(var site: String, var category: SchedulerContext.COMMAND_TYPE.TYPE)
-case class Terminated(context: SchedulerContext)
-
-class CommandCoordinator extends UntypedActor with ActorLogging{
- var consumer:ActorRef = null
- var ondemandTrainingProducer:ActorRef = null
- var periodicTrainingProducer:ActorRef = null
-
- override def preStart(): Unit = {
- consumer = context.actorOf(Props[UserProfileCommandConsumer],"userprofile-command-consumer")
- ondemandTrainingProducer = context.actorOf(Props[OndemandTrainingProducer],"persisted-command-producer")
- periodicTrainingProducer = context.actorOf(Props[PeriodicTrainingProducer],"scheduled-command-producer")
- }
-
- override def onReceive(message: Any): Unit = message match {
- case Initialized(config) =>
- log.info(s"Config updated: $config")
- ondemandTrainingProducer ! config
- periodicTrainingProducer ! config
- consumer ! config
- case request: CheckOndemandTrainingStatus =>
- ondemandTrainingProducer ! request
- case request: CheckPeriodicTrainingStatus =>
- periodicTrainingProducer ! request
- case command: Command =>
- log.info(s"*** NEW Command: $command ***")
- consumer ! command
- case Terminated(config) =>
- log.info("Coordinator exited")
- case _ =>
- log.warning(s"Unhandled message: $message")
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Producers.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Producers.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Producers.scala
deleted file mode 100644
index fbf917d..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Producers.scala
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import akka.actor.{ActorLogging, ActorRef, UntypedActor}
-import akka.event.LoggingAdapter
-import org.apache.eagle.common.DateTimeUtil
-import org.apache.eagle.security.userprofile.daemon.SchedulerContext.{COMMAND_SOURCE, COMMAND_TYPE}
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity.STATUS
-
-import scala.collection.JavaConversions
-import scala.util.{Failure, Success}
-
-/**
- * @since 9/10/15
- */
-
-object CommandProducer{
- def toStateName(site:String,category:COMMAND_TYPE.TYPE):String = s"$site-${category.toString.toLowerCase}"
-
- def getProgramStatus(site:String, category:COMMAND_TYPE.TYPE,policy: SchedulePolicy):Option[Long] = policy.getStatus(toStateName(site,category))
-
- def checkProgramStatus(site:String, category:COMMAND_TYPE.TYPE,policy: SchedulePolicy,inputPath:String, sender: ActorRef, log: LoggingAdapter)(buildCommand: (Long,String) => UserProfileCommand): Unit = {
- val stateName = toStateName(site,category)
- var status: Long = policy.getStatus(stateName).getOrElse(0l)
- if (status > 0) {
- status = policy.formatStatus(status)
- if (log.isDebugEnabled) log.debug(s"State [$stateName] is found: [${DateTimeUtil.millisecondsToHumanDateWithSeconds(status)}]")
- if (!policy.validateStatus(status)) {
- if (log.isDebugEnabled) {
- val nextStatus = policy.nextStatus(status)
- log.debug(s"State [$stateName]: [${DateTimeUtil.millisecondsToHumanDateWithSeconds(status)}], will execute at [${DateTimeUtil.millisecondsToHumanDateWithSeconds(nextStatus)}]")
- }
- return
- }
- } else {
- status = policy.formatStatus(System.currentTimeMillis())
- log.info(s"Initialized State [$stateName] as current timestamp by default: $status")
- policy updateStatus(stateName, status)
- // return
- }
-
- val path = Utils.formatPathWithMilliseconds(inputPath)(status)
- log.info(s"Create a new command for [status: ${DateTimeUtil.millisecondsToHumanDateWithSeconds(status)}, path: $path]")
- sender ! buildCommand(status,path)
- val nextStatus = policy nextStatus status
- policy updateStatus(stateName, nextStatus)
- }
-}
-
-class OndemandTrainingProducer extends UntypedActor with ActorLogging{
- @volatile var dao:UserProfileCommandDao=null
- @volatile var config:SchedulerContext=null
-
- import context.dispatcher
-
- override def onReceive(message: Any): Unit = message match {
- case _config: SchedulerContext =>
- if(log.isDebugEnabled) {
- log.debug(s"Initialized with config: ${_config}")
- } else {
- if(config == null) log.info(s"Initialized") else log.info("Reinitialized")
- }
- config = _config
- dao = new UserProfileCommandDao(config.eagleServiceContext.serviceHost,config.eagleServiceContext.servicePort,config.eagleServiceContext.username,config.eagleServiceContext.password,this)
- case CheckOndemandTrainingStatus(site,category) =>
- val _sender = sender()
- dao.readNewInitializedCommandByType(site,category) onComplete {
- case Success(optionalEntities) =>
- optionalEntities match {
- case Some(entities) =>
- log.info(s"Load ${entities.size} new $category commands")
- JavaConversions.collectionAsScalaIterable(entities) foreach { entity =>
- dao.updateCommandStatus(entity,STATUS.PENDING,"Command is pending to execute") onComplete {
- case Success(response) =>
- if(response.isSuccess) {
- _sender ! entity2Command(site, category, entity, config)
- }else{
- log.error(s"Got exception to update status as PENDING for command:$entity, due to service exception: ${response.getException}")
- }
- case Failure(ex) =>
- log.error(ex, s"Got exception to update status as PENDING for command:$entity, due to: ${ex.getMessage}")
- }
- }
- case None =>
- if(log.isDebugEnabled) {
- log.debug(s"Loaded 0 new $category commands")
- }
- }
- case Failure(exception:Exception) => {
- log.error(exception, s"Failed to get commands for site = [$site] and category = [$category], due to ${exception.getMessage}")
- }
- }
- }
-
- private def entity2Command(site:String,category:COMMAND_TYPE.TYPE,entity: ScheduleCommandEntity,config:SchedulerContext):Command = {
- category match {
- case COMMAND_TYPE.USER_PROFILE_TRAINING =>
- val policy = config.trainingSchedulePolicy
- val status = entity.getTimestamp
- val path = Utils.formatPathWithMilliseconds(config.trainingAuditPath)(status)
- UserProfileModelCommand(entity.getTimestamp,path,site,STATUS.INITIALIZED,COMMAND_SOURCE.ONDEMAND,entity.getTimestamp,entity.getUpdateTime,persistable = true)
- case _ => throw new IllegalArgumentException(s"Unknown type of command type: $category")
- }
- }
-}
-
-class PeriodicTrainingProducer extends UntypedActor with ActorLogging{
- @volatile var config:SchedulerContext = null
- @volatile var dao:UserProfileCommandDao=null
-
- import CommandProducer._
-
- def checkTrainingProgramStatus(site: String,category:COMMAND_TYPE.TYPE,sender: ActorRef): Unit = {
- checkProgramStatus(site,category,config.trainingSchedulePolicy,config.trainingAuditPath,sender,log) {(status,path) =>
- UserProfileModelCommand(status,path,site,STATUS.INITIALIZED,COMMAND_SOURCE.PERIODIC,System.currentTimeMillis(),System.currentTimeMillis(),persistable = false)
- }
- }
-
- override def onReceive(message: Any): Unit = message match {
- case _config: SchedulerContext =>
- if (log.isDebugEnabled) {
- log.debug(s"Initialized with config: $config")
- } else {
- if(config == null) log.info(s"Initialized") else log.info("Reinitialized")
- }
- config = _config
- dao = new UserProfileCommandDao(config.eagleServiceContext.serviceHost,config.eagleServiceContext.servicePort,config.eagleServiceContext.username,config.eagleServiceContext.password,this)
- case CheckPeriodicTrainingStatus(site,category) =>
- category match {
- case COMMAND_TYPE.USER_PROFILE_TRAINING =>
- log.debug("Checking training program status")
- checkTrainingProgramStatus(site,category,sender())
- case _ =>
- log.error(s"Unknown type of category: $category")
- }
- case _ => throw new IllegalArgumentException(s"Unknown message $message")
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulePolicy.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulePolicy.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulePolicy.scala
deleted file mode 100644
index 2dde4ce..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulePolicy.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import org.apache.eagle.security.userprofile.UserProfileUtils
-import org.joda.time.{LocalDate, Period}
-
-import scala.collection.mutable
-
-/**
- * User Profile Daemon Schedule Policy
- * Configure scheduler by defining policy for managing status
- *
- * @since 9/5/15
- */
-trait SchedulePolicy{
- /**
- * @param timestamp
- * @return
- */
- def validateStatus(timestamp:Long):Boolean
-
- /**
- *
- * @param timestamp
- * @return
- */
- def nextStatus(timestamp:Long):Long
-
- /**
- *
- * @param name
- * @return
- */
- def getStatus(name:String):Option[Long]
-
- /**
- *
- * @param name
- * @param status
- */
- def updateStatus(name:String,status:Long):Unit
-
- /**
- *
- * @param timestamp
- * @return
- */
- def formatStatus(timestamp:Long):Long
-}
-
-trait StateManager {
- def getState(name:String):Option[Long]
- def updateState(name:String,value:Long)
-}
-
-object MemoryStateManager extends StateManager {
- val map = new mutable.HashMap[String,Long]()
- override def getState(name: String): Option[Long] = this.synchronized { map.get(name) }
- override def updateState(name: String, value: Long): Unit = this.synchronized {
- map += (name -> value)
- }
-}
-
-object ZKStateManager extends StateManager{
- override def getState(name: String): Option[Long] = ???
- override def updateState(name: String, value: Long): Unit = ???
-}
-
-class DefaultSchedulePolicy(duration:Period,stateManager:StateManager = MemoryStateManager) extends SchedulePolicy{
- override def validateStatus(timestamp: Long): Boolean = formatStatus(timestamp) < formatStatus(System.currentTimeMillis())
-
-// override def nextStatus(timestamp: Long): Long = new LocalDate(timestamp).plus(duration).toDate.getTime
- override def nextStatus(timestamp: Long): Long = timestamp + Int.int2long(duration.toStandardSeconds.getSeconds) * 1000
-
- override def getStatus(name: String): Option[Long] = stateManager.getState(name)
-
- override def updateStatus(name: String, status: Long): Unit = stateManager.updateState(name,status)
-
- override def formatStatus(timestamp: Long): Long = UserProfileUtils.formatMillisecondsByPeriod(timestamp,duration)
-}
-
-class MonthlySchedulePolicy(stateManager:StateManager = MemoryStateManager) extends DefaultSchedulePolicy(duration = null,stateManager=stateManager){
- override def formatStatus(timestamp: Long): Long = new LocalDate(timestamp).withDayOfMonth(1).toDate.getTime
- override def nextStatus(timestamp: Long): Long = new LocalDate(timestamp).plusMonths(1).withDayOfMonth(1).toDate.getTime
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Scheduler.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Scheduler.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Scheduler.scala
deleted file mode 100644
index 770ad6a..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Scheduler.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import akka.actor.{ActorSystem, Props}
-import SchedulerContext.COMMAND_TYPE
-import org.apache.eagle.dataproc.util.ConfigOptionParser
-
-import scala.concurrent.duration._
-
-/**
- * User Profile Training Scheduler
- *
- * @since 0.3.0
- */
-class Scheduler(config:SchedulerContext) {
- /**
- * Start Daemon Scheduler
- */
- def start(): Unit = {
- val system = ActorSystem(SchedulerContext.SchedulerSystem)
-
- system.log.info(s"Started actor system: $system")
-
- import system.dispatcher
-
- val coordinator = system.actorOf(Props[CommandCoordinator])
- // Initialize when start
- system.scheduler.scheduleOnce(0.seconds,coordinator,Initialized(config))
-
- system.scheduler.schedule(1.seconds,config.syncIntervalSeconds.seconds,coordinator,CheckOndemandTrainingStatus(config.site,COMMAND_TYPE.USER_PROFILE_TRAINING))
-
- system.scheduler.schedule(config.trainingInitialDelaySeconds.seconds,config.trainingIntervalSeconds.seconds,coordinator,CheckPeriodicTrainingStatus(config.site,COMMAND_TYPE.USER_PROFILE_TRAINING))
-
- system.registerOnTermination(new Runnable {
- override def run(): Unit = {
- coordinator ! Terminated(config)
- }
- })
- }
-}
-
-object Scheduler{
- def main(args:Array[String]): Unit ={
- new ConfigOptionParser().load(args)
- new Scheduler(SchedulerContext.load).start()
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulerContext.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulerContext.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulerContext.scala
deleted file mode 100644
index fcc64fc..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/SchedulerContext.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import java.net.URLDecoder
-
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.eagle.security.userprofile.UserProfileTrainingCLI
-
-object SchedulerContext {
- val SchedulerSystem = "userprofile-scheduler-system"
- val UserProfileTrainingJarFilePath = URLDecoder.decode(classOf[UserProfileTrainingCLI].getProtectionDomain.getCodeSource.getLocation.getPath,"UTF-8")
- val UserProfileTrainingCLIClass = classOf[UserProfileTrainingCLI].getCanonicalName
- val HdfsAuditLogAggStatus = "hdfs.audit.agg.status"
- val HdfsAuditLogModelStatus = "hdfs.audit.model.status"
-
- // {
- // ======================================================================================
- // Configuration Keys
- // ======================================================================================
- val UserProfileConfigKey_Site="eagle.site"
- val UserProfileConfigKey_DryRun="eagle.userprofile.dry-run"
-
-
- val UserProfileConfigKey_Features="eagle.userprofile.features"
- val UserProfileConfigKey_Period="eagle.userprofile.period"
-
- val UserProfileConfigKey_JobJarFilePath="eagle.userprofile.jar"
- val UserProfileConfigKey_DriverShell="eagle.userprofile.driver-shell"
- val UserProfileConfigKey_DriverClasspath="eagle.userprofile.driver-classpath"
- val UserProfileConfigKey_SparkMaster="eagle.userprofile.spark-master"
- val UserProfileConfigKey_SparkMode="eagle.userprofile.spark-mode"
-
- val UserProfileConfigKey_ModelAuditPath="eagle.userprofile.training-audit-path"
- val UserProfileConfigKey_ModelIntervalSeconds="eagle.userprofile.training-interval-seconds"
- val UserProfileConfigKey_ModelInitialDelaySeconds="eagle.userprofile.training-initial-delay-seconds"
- val UserProfileConfigKey_ServiceHost="eagle.service.host"
- val UserProfileConfigKey_ServicePort="eagle.service.port"
- val UserProfileConfigKey_UserName="eagle.service.username"
- val UserProfileConfigKey_Password="eagle.service.password"
- val UserProfileConfigKey_SyncIntervalSeconds="eagle.userprofile.sync-interval-seconds"
- // ======================================================================================
- // End of Configuration Keys
- // ======================================================================================
- // }
-
- object COMMAND_TYPE extends Enumeration {
- type TYPE = Value
- val USER_PROFILE_TRAINING = Value
- }
-
- object COMMAND_SOURCE extends Enumeration {
- type TYPE = Value
- val PERIODIC, ONDEMAND = Value
- }
-
- /**
- * Load scheduler context from configuration
- */
- def load:SchedulerContext={
- val config:Config = ConfigFactory.load()
- var site = ""
-
- if(config.hasPath(UserProfileConfigKey_Site)) site = config.getString(UserProfileConfigKey_Site)
-
- if(site == null || site.isEmpty) throw new IllegalArgumentException(s"Config [$UserProfileConfigKey_Site] is required, but not given")
-
- val context = new SchedulerContext(site)
-
- if(config.hasPath(UserProfileConfigKey_DryRun)) context.dryRun = config.getBoolean(UserProfileConfigKey_DryRun)
- if(config.hasPath(UserProfileConfigKey_Features)) context.features = config.getString(UserProfileConfigKey_Features)
- if(config.hasPath(UserProfileConfigKey_Period)) context.period = config.getString(UserProfileConfigKey_Period)
-
- if(config.hasPath(UserProfileConfigKey_ModelAuditPath)) {
- context.trainingAuditPath = config.getString(UserProfileConfigKey_ModelAuditPath)
- }else{
- throw new IllegalArgumentException(s"Config[$UserProfileConfigKey_ModelAuditPath] should not be null")
- }
-
- if(config.hasPath(UserProfileConfigKey_ModelIntervalSeconds)) context.trainingIntervalSeconds = config.getLong(UserProfileConfigKey_ModelIntervalSeconds)
- if(config.hasPath(UserProfileConfigKey_ModelInitialDelaySeconds)) context.trainingInitialDelaySeconds = config.getLong(UserProfileConfigKey_ModelInitialDelaySeconds)
-
- if(config.hasPath(UserProfileConfigKey_ServiceHost)) context.eagleServiceContext.serviceHost = config.getString(UserProfileConfigKey_ServiceHost)
- if(config.hasPath(UserProfileConfigKey_ServicePort)) context.eagleServiceContext.servicePort = config.getInt(UserProfileConfigKey_ServicePort)
- if(config.hasPath(UserProfileConfigKey_UserName)) context.eagleServiceContext.username = config.getString(UserProfileConfigKey_UserName)
- if(config.hasPath(UserProfileConfigKey_Password)) context.eagleServiceContext.password = config.getString(UserProfileConfigKey_Password)
-
- if(config.hasPath(UserProfileConfigKey_JobJarFilePath)) context.jobJar = config.getString(UserProfileConfigKey_JobJarFilePath)
-
- if(config.hasPath(UserProfileConfigKey_SparkMaster)) context.sparkMaster = config.getString(UserProfileConfigKey_SparkMaster)
- if(config.hasPath(UserProfileConfigKey_SparkMode)) context.sparkMode = config.getString(UserProfileConfigKey_SparkMode)
- if(config.hasPath(UserProfileConfigKey_DriverShell)) context.driverShell = config.getString(UserProfileConfigKey_DriverShell)
- if(config.hasPath(UserProfileConfigKey_DriverClasspath)) context.driverClasspath = config.getString(UserProfileConfigKey_DriverClasspath)
- if(config.hasPath(UserProfileConfigKey_SyncIntervalSeconds)) context.syncIntervalSeconds = config.getLong(UserProfileConfigKey_SyncIntervalSeconds)
-
- context
- }
-}
-
-case class EagleServiceContext(
- var serviceHost:String = "localhost",
- var servicePort:Int = 9099,
- var username:String = "admin",
- var password:String = "secure"
-)
-
-case class SchedulerContext(
- var site:String,
- var dryRun:Boolean = true,
- var features:String = null,
- var period:String = "PT1m", // changing to 1m aggregation
- var eagleServiceContext: EagleServiceContext=new EagleServiceContext(),
- var jobJar: String = SchedulerContext.UserProfileTrainingJarFilePath,
- var driverShell: String = "spark-submit",
- var driverClasspath: String = null,
- var sparkMaster:String = "local[10]",
- var sparkMode:String = "client",
- var trainingAuditPath:String = null,
- var trainingIntervalSeconds:Long = 60,
- var trainingInitialDelaySeconds:Long = 0,
- /**
- * Detection sync commands interval in seconds
- */
- var syncIntervalSeconds:Long = 5,
- /**
- * Training program schedule policy
- */
- trainingSchedulePolicy:SchedulePolicy = new MonthlySchedulePolicy()
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/UserProfileCommandDao.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/UserProfileCommandDao.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/UserProfileCommandDao.scala
deleted file mode 100644
index a129074..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/UserProfileCommandDao.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import java.util
-import java.util.concurrent.Callable
-
-import akka.actor.UntypedActor
-import akka.dispatch.Futures
-import akka.event.Logging
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity
-import org.apache.eagle.security.userprofile.daemon.SchedulerContext.COMMAND_TYPE
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity
-import org.apache.eagle.security.userprofile.model.ScheduleCommandEntity.STATUS
-import org.apache.eagle.service.client.impl.EagleServiceClientImpl
-
-/**
- * User Profile Command Dao By Eagle Service
- * @since 9/11/15
- */
-class UserProfileCommandDao(host:String,port:Int,username:String,password:String,actor:UntypedActor){
- val client = new EagleServiceClientImpl(host,port,username,password)
- val log = Logging.getLogger(actor)
-
- /**
- * readNewInitializedCommandByType
- *
- * @param site eagle site name
- * @param commandType command type
- * @return Future
- */
- def readNewInitializedCommandByType(site:String,commandType: COMMAND_TYPE.TYPE) = {
- Futures.future(new Callable[Option[java.util.List[ScheduleCommandEntity]]]{
- override def call(): Option[java.util.List[ScheduleCommandEntity]] = {
- val response:GenericServiceAPIResponseEntity[ScheduleCommandEntity] =
- new EagleServiceClientImpl(host,port,username,password)
- .silence(true)
- .search(s"${ScheduleCommandEntity.ScheduleTaskService}[@site = ${"\"" + site +"\""} AND @type = ${"\""+commandType.toString+"\""} AND (@status = ${"\""+STATUS.INITIALIZED.name()+"\""} or @status = ${"\"\""} or @status is null)]{*}")
- .startTime(0).endTime(Long.MaxValue).pageSize(Int.MaxValue).send()
-
- if(!response.isSuccess){
- throw new RuntimeException(s"Got server side exception: ${response.getException}")
- }
-
- if(response.getObj != null && response.getObj.size() == 0) None else Option(response.getObj)
- }
- },actor.getContext().dispatcher)
- }
-
- /**
- * updateCommandStatus
- *
- * @param task task entity
- * @param status task status
- * @param message message string
- * @return
- */
- def updateCommandStatus(task:ScheduleCommandEntity,status:STATUS,message:String) ={
- Futures.future(new Callable[GenericServiceAPIResponseEntity[String]] {
- override def call(): GenericServiceAPIResponseEntity[String] = {
- if(log.isDebugEnabled) log.debug(s"Updating status of task[$task] as: $status")
- task.setStatus(status.name())
- task.setDetail(s"[${status.name()}] $message")
- task.setUpdateTime(System.currentTimeMillis())
- val response = client.update(util.Arrays.asList(task),classOf[ScheduleCommandEntity])
- if(response.isSuccess){
- log.info(s"Updated status of command [$task] as: $status")
- } else {
- log.error(s"Failed to update status of command [$task] as: $status, because of exception: ${response.getException}")
- throw new RuntimeException(s"Failed to update status due to exception: ${response.getException}")
- }
- response
- }
- },actor.getContext().dispatcher)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Utils.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Utils.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Utils.scala
deleted file mode 100644
index 1a0f3bb..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/daemon/Utils.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.daemon
-
-import java.util.regex.Pattern
-import org.apache.eagle.common.DateTimeUtil
-import org.apache.eagle.security.userprofile.UserProfileUtils
-import org.joda.time.Period
-
-import scala.collection.mutable
-
-/**
- * @since 9/2/15
- */
-object Utils {
- val PATH_TIME_FORMAT_PATTERN = Pattern.compile("\\$\\{([\\*\\/\\w\\-_\\s]+)\\}")
-
- /**
- * Example: hdfs:///logs/nn/auditlog/${yyyy-mm-dd}/hdfs-audit.log.${yyyy-mm-dd-hh}.gz
- *
- * @param path
- * @return
- */
- def formatPathWithMilliseconds(path:String)(milliseconds:Long):String = {
- val matcher = PATH_TIME_FORMAT_PATTERN.matcher(path)
- var result = path
- while(matcher.find()){
- val grouped = matcher.group()
- val format = grouped.substring(2,grouped.length()-1)
- val formatted = DateTimeUtil.format(milliseconds,format)
- result = result.replace(grouped,formatted)
- }
- result
- }
-
- /**
- * For example:
- *
- * current: 2015/09/02 10:12:32
- * granularity: 15m
- * duration: 1M // changing it to 1M interval of aggregation
- *
- * 1) flatten to duration: 2015/09/02 10:12:32 => 2015/09/02 10:00:00
- * 2) split by granularity: 2015/09/02 10:00:00 / 15m => [2015/09/02 10:00:00,2015/09/02 10:15:00,2015/09/02 10:30:00,2015/09/02 10:45:00]
- * 3) format file paths
- */
- def formatPathsInDuration(pathFormat:String,milliseconds:Long,duration:Period,granularity: Period = Period.parse("PT1M")): Seq[String] = {
- val durationIsSeconds = duration.toStandardSeconds
- val durationInMs = Int.int2long(durationIsSeconds.getSeconds) * 1000
- val granularityInMs = Int.int2long(granularity.toStandardSeconds.getSeconds) * 1000
-
- val flattenTimestamp =UserProfileUtils.formatSecondsByPeriod(milliseconds/1000,durationIsSeconds) * 1000
- val timeSplits:mutable.MutableList[Long] = new mutable.MutableList[Long]()
- var tmp = flattenTimestamp
-
- while(tmp - flattenTimestamp < durationInMs){
- timeSplits += tmp
- tmp = tmp + granularityInMs
- }
-
- timeSplits.map(formatPathWithMilliseconds(pathFormat))
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/AuditLogTrainingSparkJob.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/AuditLogTrainingSparkJob.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/AuditLogTrainingSparkJob.scala
deleted file mode 100644
index 6253351..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/AuditLogTrainingSparkJob.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.job
-
-import org.apache.commons.math3.linear.{Array2DRowRealMatrix, RealMatrix}
-import org.apache.eagle.security.userprofile.UserProfileConstants
-import org.apache.eagle.security.userprofile.model.AuditLogTransformer
-import org.apache.spark.storage.StorageLevel
-import org.apache.spark.{Logging, SparkConf, SparkContext}
-import org.joda.time.Period
-
-import scala.collection.mutable
-
-/**
- * Audit Log UserProfileTrainingJob Implementation
- *
- * @param site site
- * @param input input path
- * @param master spark master
- * @param appName spark application name
- * @param cmdTypes command features
- * @param period model aggregation granularity
- */
-case class AuditLogTrainingSparkJob(
- site:String = null,
- var input:String=null,
- master:String="local[4]",
- appName:String=UserProfileConstants.DEFAULT_TRAINING_APP_NAME,
- cmdTypes:Seq[String] = UserProfileConstants.DEFAULT_CMD_TYPES,
- period:Period = Period.parse("PT1m")
- ) extends UserProfileTrainingJob with Logging {
- import org.apache.eagle.security.userprofile.job.AuditlogTrainingSparkJobFuncs._
-
- val conf = new SparkConf().setMaster(master).setAppName(appName)
- val sc = new SparkContext(conf)
-
- override def run(){
- logInfo(s"Starting $appName")
- buildDAG(sc)
- logInfo(s"Stopping $appName")
- sc.stop()
- logInfo(s"Finished $appName")
- }
-
- /**
- * Build Spark DAG
- *
- * @param sc
- */
- private def buildDAG(sc:SparkContext): Unit = {
- if (input == null) throw new IllegalArgumentException("input is null")
-
- val _site = site
- val _sc = sc
- val _cmdTypes = cmdTypes
- val _modelers = modelers
- val _modelSinks = modelSinks
- val _aggSinks = aggSinks
-
- val _period = period
- val tmp = _sc.textFile(input)
- .map(AuditLogTransformer(_period).transform)
- .filter(e => e.isDefined)
- .map(e => {((e.get.user,e.get.periodWindowSeconds,e.get.cmd),1.toDouble) }) // [(user,period,cmd),count]
-
- val _aggRDD = tmp
- .reduceByKey(_ + _) // [(user,period,cmd),totalCount]
- .map(kv => {((kv._1._1, kv._1._2), (kv._1._3, kv._2))}) // [(user,period),(cmd,totalCount)]
- .groupByKey()
- .map(asUserCmdCountArray(_cmdTypes)) // [(user,(period,array))]
- .sortBy(asUserPeriod, ascending = true)
- .groupByKey()
- .map(asMatrix)
-
- if (_modelers.size > 1 || _aggSinks.size > 1) _aggRDD.persist(StorageLevel.MEMORY_ONLY)
-
- _aggSinks.foreach(sink =>{
- sink.persist(_aggRDD,_cmdTypes,_site)
- })
-
- val modelRDDS = _modelers.map(
- modeler => {
- (_aggRDD.flatMap(v => {
- val _modeler = modeler
- _modeler.build(_site,v._1, v._2)
- }), modeler.context())
- })
-
- if(_modelSinks.size > 1) modelRDDS.foreach(_._1.cache())
-
- modelRDDS.foreach(kv => {
- _modelSinks.foreach(sink => {
- sink.persist(kv._1, kv._2)
- })
- })
- }
-
- def input(path:String):AuditLogTrainingSparkJob = {
- this.input = path
- this
- }
-}
-
-private object AuditlogTrainingSparkJobFuncs extends Logging{
- /**
- * From: [(user,period),(cmd,count)]
- *
- * To: [(user), (period,"getfileinfo", "open", "listStatus", "setTimes", "setPermission", "rename", "mkdirs", "create", "setReplication", "contentSummary", "delete", "setOwner", "fsck")]
- *
- * @param keyValue
- * @return
- */
- def asUserCmdCountArray(cmdTypes:Seq[String])(keyValue:(((String,Long),Iterable[(String,Double)])))={
- // make sure instance variables are always in local for closure
- val key = keyValue._1
- val it = keyValue._2
- var cmdCount = Map[String, Double]()
- it.foreach(k => {
- cmdCount += (k._1 -> k._2)
- })
- var cmdCounts = mutable.MutableList[Double]()
- cmdTypes.foreach(_cmdType => {
- cmdCounts += cmdCount.getOrElse[Double](_cmdType, 0.0)
- })
- val cmdCountsInDouble:Array[Double] = cmdCounts.toArray
- (key._1,(key._2,cmdCountsInDouble))
- }
-
- /**
- * @param pair
- * @return (user,matrix)
- */
- def asMatrix(pair:(String,Iterable[(Long,Array[Double])]))={
- val data = new mutable.MutableList[Array[Double]]
- pair._2.foreach(v => data += v._2)
- val matrix:RealMatrix = new Array2DRowRealMatrix(data.toArray)
- (pair._1,matrix)
- }
-
- def asUserPeriod(param:(String,(Long,Array[Double]))) ={
- (param._1, param._2._1)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/UserProfileTrainingJob.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/UserProfileTrainingJob.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/UserProfileTrainingJob.scala
deleted file mode 100644
index aae11a0..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/job/UserProfileTrainingJob.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.job
-
-import org.apache.eagle.security.userprofile.model.{UserProfileModel, UserProfileContext, UserProfileModeler}
-import org.apache.eagle.security.userprofile.sink.{UserActivityAggRDDSink, UserProfileModelRDDSink}
-
-import scala.collection.mutable
-
-/**
- * User Profile Training Job Interface
- *
- * @since 7/19/15
- */
-trait UserProfileTrainingJob {
- /**
- * Start the app
- *
- * @return
- */
- def run()
-
- /**
- * Register training modeler
- *
- * @param modeler
- */
- def model(modeler:UserProfileModeler[UserProfileModel,UserProfileContext]): UserProfileTrainingJob = {
- this.modelers += modeler
- this
- }
-
- /**
- * Register model data consumer
- * @param sink
- */
- def sink(sink: UserProfileModelRDDSink): UserProfileTrainingJob = {
- this.modelSinks += sink
- this
- }
-
- def sink(sink: UserActivityAggRDDSink): UserProfileTrainingJob = {
- this.aggSinks += sink
- this
- }
-
- val modelers = new mutable.MutableList[UserProfileModeler[UserProfileModel,UserProfileContext]]()
- val modelSinks = new mutable.MutableList[UserProfileModelRDDSink]()
- val aggSinks = new mutable.MutableList[UserActivityAggRDDSink]()
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileContext.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileContext.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileContext.scala
deleted file mode 100644
index ea84315..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileContext.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-/**
- * @since 0.3.0
- */
-case class UserProfileContext(algrithm:String)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModeler.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModeler.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModeler.scala
deleted file mode 100644
index de52e73..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/model/UserProfileModeler.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model
-
-import org.apache.commons.math3.linear.RealMatrix
-
-/**
- * @since 0.3.0
- */
-trait UserProfileModeler[+M,+C <: UserProfileContext] extends Serializable{
- /**
- * @param site site
- * @param user user
- * @param matrix user profile matrix
- */
- def build(site:String,user:String,matrix: RealMatrix):List[M]
-
- /**
- * @return
- */
- def context():C
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserActivityAggSink.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserActivityAggSink.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserActivityAggSink.scala
deleted file mode 100644
index bd16ba9..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserActivityAggSink.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.sink
-
-import java.util.Properties
-
-import kafka.producer.{KeyedMessage, Producer, ProducerConfig}
-import org.apache.commons.math3.linear.RealMatrix
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity
-import org.apache.eagle.security.userprofile.model.{EntityConversion, UserActivityAggModel}
-import org.apache.spark.Logging
-import org.apache.spark.rdd.RDD
-import org.codehaus.jackson.map.ObjectMapper
-
-/**
- * @since 0.3.0
- */
-trait UserActivityAggRDDSink {
- def persist(rdd: RDD[(String, RealMatrix)],cmdTypes:Seq[String],site:String): Unit
-}
-
-abstract class UserActivityAggModelSink extends UserActivityAggRDDSink{
- override def persist(rdd: RDD[(String, RealMatrix)],cmdTypes:Seq[String],site:String): Unit = {
- rdd.foreach(kv => {
- this.persist(UserActivityAggModel(kv._1,kv._2,cmdTypes,site,System.currentTimeMillis()))
- })
- }
-
- def persist(model: UserActivityAggModel): Unit
-}
-
-case class UserActivityAggKafkaSink(properties: Properties) extends UserActivityAggModelSink with Logging{
-
- @transient var producer:Producer[String, String] = null
- @transient var objectMapper:ObjectMapper = null
-
- val TOPIC_KEY = "topic"
- val topicName = properties.getProperty(TOPIC_KEY)
- if(topicName == null) throw new IllegalArgumentException(s"$TOPIC_KEY is null")
-
- private[this] def getProducer = {
- if(producer == null){
- if(!properties.containsKey("serializer.class")) properties.setProperty("serializer.class", "kafka.serializer.StringEncoder")
- if(!properties.containsKey("metadata.broker.list")) properties.setProperty("metadata.broker.list", "localhost:9092")
- val config = new ProducerConfig(properties)
- producer = new Producer[String,String](config)
- }
- producer
- }
-
- private[this] def getObjectMapper = {
- if(objectMapper == null) {
- objectMapper = TaggedLogAPIEntity.buildObjectMapper()
- }
- objectMapper
- }
-
- override def persist(model: UserActivityAggModel): Unit = {
- val entity = model.asInstanceOf[EntityConversion[TaggedLogAPIEntity]].toEntity
- val message = new KeyedMessage[String, String](topicName, model.user, getObjectMapper.writeValueAsString(entity))
- try {
- getProducer.send(message)
- }catch {
- case e:Exception => {
- logError(s"Failed to send message to kafka[$properties]",e)
- throw e
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserProfileModelSink.scala
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserProfileModelSink.scala b/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserProfileModelSink.scala
deleted file mode 100644
index d6fbae9..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/scala/org/apache/eagle/security/userprofile/sink/UserProfileModelSink.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.sink
-
-import java.util
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.apache.eagle.security.userprofile.model.{EntityConversion, UserProfileContext, UserProfileModel}
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity
-import org.apache.eagle.service.client.IEagleServiceClient
-import org.apache.eagle.service.client.impl.EagleServiceClientImpl
-import org.apache.spark.Logging
-import org.apache.spark.rdd.RDD
-
-trait UserProfileModelRDDSink extends Serializable{
- def persist(model: RDD[UserProfileModel],context: UserProfileContext)
-}
-
-abstract class UserProfileModelSink extends UserProfileModelRDDSink{
- def persist(model: UserProfileModel)
- override def persist(model: RDD[UserProfileModel],context:UserProfileContext): Unit = {
- model.foreach(persist)
- }
-}
-
-case class UserProfileHDFSSink(path:String) extends UserProfileModelRDDSink with Logging{
- override def persist(model: RDD[UserProfileModel],ctx: UserProfileContext): Unit = {
- val finalPath = if(path.endsWith("/")) String.format("%s%s",path,ctx.algrithm) else String.format("%s/%s",path,ctx.algrithm)
- logInfo(s"Saving to output path: $finalPath, algrithm: ${ctx.algrithm}")
- model.map(e => {
- val entity = e.asInstanceOf[EntityConversion[TaggedLogAPIEntity]].toEntity
- entity.setSerializeVerbose(false)
- TaggedLogAPIEntity.buildObjectMapper().writeValueAsString(entity)
- }).saveAsTextFile(finalPath)
- }
-}
-
-case class UserProfileEagleServiceSink(host:String = "localhost",port:Int = 9099, username:String = "admin", password:String = "secret", maxRetryTimes:Int = 100) extends UserProfileModelSink with Logging{
- @transient var client:IEagleServiceClient = null
-
- def persist(model: UserProfileModel): Unit = {
- if (model == null) {
- logDebug("model is null")
- return
- }
- if (client == null) client = new EagleServiceClientImpl(host, port, username, password)
- var triedTimes = 0
- var response: GenericServiceAPIResponseEntity[String] = null
- while (triedTimes < maxRetryTimes && (response == null || !response.isSuccess)) {
- triedTimes += 1
- try{
- response = client.create(util.Arrays.asList(model.asInstanceOf[EntityConversion[TaggedLogAPIEntity]].toEntity))
- if (response.isSuccess) {
- logInfo(s"Successfully persist 1 model for user: ${model.user} algorithm: ${model.algorithm}")
- } else {
- logError(s"Failed to persist 1 model for user: ${model.user} algorithm: ${model.algorithm}, tried $triedTimes times, exception: ${response.getException}")
- }
- } catch {
- case e: Throwable => {
- logError(s"Failed to persist for exception 1 model for user: ${model.user} algorithm: ${model.algorithm}, tried $triedTimes times, exception: ${e.getMessage}",e)
- response = null
- }
- }
- }
- }
-}
-
-case object UserProfileStdoutAsJsonSink extends UserProfileModelSink with Logging{
- val objMapper = new ObjectMapper
- override def persist(model: UserProfileModel): Unit = {
- val json = objMapper.writeValueAsString(model)
- println(json)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelerTest.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelerTest.java b/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelerTest.java
deleted file mode 100644
index 3d5d6a6..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/test/java/org/apache/eagle/security/userprofile/model/UserProfileEigenModelerTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.eagle.security.userprofile.model.eigen.UserProfileEigenModeler;
-import org.junit.Assert;
-import scala.collection.immutable.List;
-
-public class UserProfileEigenModelerTest {
-
- @org.junit.Test
- public void testBuild() throws Exception {
-
- UserProfileEigenModeler modeler = new UserProfileEigenModeler();
- String user = "user1";
- final RealMatrix mockMatrix = new Array2DRowRealMatrix(buildMockData());
- List<UserProfileEigenModel> model = modeler.build("default",user, mockMatrix);
- Assert.assertEquals(model.length(), 1);
-
- UserProfileEigenModel eigenModel = model.head();
- Assert.assertNotNull(eigenModel.statistics());
- Assert.assertNotNull(eigenModel.principalComponents());
- Assert.assertNotNull(eigenModel.maxVector());
- Assert.assertNotNull(eigenModel.minVector());
- Assert.assertEquals(eigenModel.statistics().length, mockMatrix.getColumnDimension());
- Assert.assertTrue(eigenModel.principalComponents().length <= mockMatrix.getColumnDimension());
- Assert.assertTrue(eigenModel.maxVector().getDimension() <= mockMatrix.getColumnDimension());
- Assert.assertTrue(eigenModel.minVector().getDimension() <= mockMatrix.getColumnDimension());
- Assert.assertEquals(true, eigenModel.statistics()[3].isLowVariant());
- }
-
- private double[][] buildMockData(){
- double[][] mockData = new double[][]{
- {114503.0,2.8820906E7,123618.0,0.0,64.0,15772.0,186.0,24296.0,12.0,9.0,32.0,0.0,0.0},
- {53300.0,390772.0,157626.0,0.0,67.0,10501.0,226.0,8567.0,14.0,0.0,12.0,0.0,0.0},
- {25659.0,140858.0,35731.0,0.0,169.0,1619.0,520.0,2965.0,34.0,0.0,34.0,0.0,0.0}
- };
- return mockData;
- }
-}
\ No newline at end of file