You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/06/10 01:29:29 UTC
[4/7] incubator-eagle git commit: EAGLE-334 clean user profile code
clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml b/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
deleted file mode 100644
index 25d079c..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>assembly</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>.</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <unpack>true</unpack>
- <scope>runtime</scope>
- <unpackOptions>
- <excludes>
- <exclude>log4j.properties</exclude>
- </excludes>
- </unpackOptions>
- <excludes>
- <!--<exclude>org.apache.spark:*</exclude>-->
- <exclude>org.apache.storm:*</exclude>
- <exclude>org.clojure:*</exclude>
- <exclude>org.scala-lang:*</exclude>
- <exclude>org.apache.commons:math3</exclude>
- <exclude>com.fasterxml.jackson.core:*</exclude>
- <exclude>com.fasterxml.jackson.module:*</exclude>
- <exclude>org.slf4j:slf4j-api:*</exclude>
- <exclude>org.slf4j:slf4j-log4j12:*</exclude>
- <exclude>log4j:log4j:*</exclude>
- <exclude>org.slf4j:log4j-over-slf4j:*</exclude>
- <exclude>org.apache.hadoop:hadoop-yarn-*</exclude>
- <exclude>org.apache.hadoop:hadoop-mapreduce-*</exclude>
- <!--<exclude>org.apache.hbase:*</exclude>-->
- </excludes>
- </dependencySet>
- </dependencySets>
- <fileSets>
- <fileSet>
- <directory>${project.build.outputDirectory}</directory>
- <outputDirectory>.</outputDirectory>
- <excludes>
- <exclude>log4j.properties</exclude>
- <exclude>application.conf</exclude>
- <exclude>hdfs-audit.log</exclude>
- </excludes>
- </fileSet>
- </fileSets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
deleted file mode 100644
index 909b7b7..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.commons.math3.linear.RealMatrix;
-import scala.collection.JavaConversions;
-import scala.collection.immutable.List;
-
-/**
- * @since 10/21/15
- */
-public abstract class JavaUserProfileModeler<M,C extends UserProfileContext> implements UserProfileModeler<M,C> {
- @Override
- public List<M> build(String site, String user, RealMatrix matrix) {
- java.util.List<M> models = generate(site, user, matrix);
- if (models != null) {
- return JavaConversions.asScalaBuffer(models).toList();
- } else {
- return null;
- }
- }
-
- /**
- * Java delegate method for {@code eagle.security.userprofile.model.JavaUserProfileModeler#build(String site, String user, RealMatrix matrix)}
- *
- * @param site eagle site
- * @param user eagle user
- * @param matrix user activity matrix
- * @return Generate user profile model
- */
- public abstract java.util.List<M> generate(String site, String user, RealMatrix matrix);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
deleted file mode 100644
index cb9d660..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.eigen;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.exception.NotStrictlyPositiveException;
-import org.apache.commons.math3.linear.*;
-import org.apache.commons.math3.stat.correlation.Covariance;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class UserProfileEigenModeler extends JavaUserProfileModeler<UserProfileEigenModel,UserProfileContext> {
-
- private final String[] cmdTypes;
- private UserCommandStatistics[] statistics;
- private String outputLocation;
- private RealMatrix finalMatrixWithoutLowVariantCmds;
- private RealMatrix covarianceMatrix;
- private RealMatrix uMatrix;
- private RealMatrix vMatrix;
- private RealMatrix diagonalMatrix;
- private int dimension =0;
- private static final double MAINTENED_VARIANCE = 0.99;
- private RealVector[] principalComponents;
- private RealVector maximumL2Norm; //Double.NEGATIVE_INFINITY;
- private RealVector minimumL2Norm; //= Double.POSITIVE_INFINITY;
- private RealVector minVector;
- private RealVector maxVector;
-
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileEigenModeler.class);
- private static final double lowVarianceVal= 0.001;
-
- public RealVector[] getPrincipalComponents() {
- return principalComponents;
- }
-
- public void setPrincipalComponents(RealVector[] principalComponents) {
- this.principalComponents = principalComponents;
- }
-
- public UserProfileEigenModeler(String[] cmdTypes){
- this.cmdTypes = cmdTypes;
- }
-
- public UserProfileEigenModeler(){
- this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
- }
-
- public RealVector getMaximumL2Norm() {
- return maximumL2Norm;
- }
-
- public void setMaximumL2Norm(RealVector maximumL2Norm) {
- this.maximumL2Norm = maximumL2Norm;
- }
-
- public RealVector getMinimumL2Norm() {
- return minimumL2Norm;
- }
-
- public void setMinimumL2Norm(RealVector minimumL2Norm) {
- this.minimumL2Norm = minimumL2Norm;
- }
-
- public RealVector getMinVector() {
- return minVector;
- }
-
- public void setMinVector(RealVector minVector) {
- this.minVector = minVector;
- }
-
- public RealVector getMaxVector() {
- return maxVector;
- }
-
- public void setMaxVector(RealVector maxVector) {
- this.maxVector = maxVector;
- }
-
- public UserCommandStatistics[] getStatistics() {
- return statistics;
- }
-
- public void setStatistics(UserCommandStatistics[] statistics) {
- this.statistics = statistics;
- }
-
- public String getOutputLocation() {
- return outputLocation;
- }
-
- public void setOutputLocation(String outputLocation) {
- this.outputLocation = outputLocation;
- }
-
- private void computeStats(RealMatrix m){
-
- if(m.getColumnDimension() != this.cmdTypes.length){
- LOG.error("Please fix the commands list in config file");
- return;
- }
- statistics = new UserCommandStatistics[m.getColumnDimension()];
- for(int i=0; i<m.getColumnDimension(); i++){
- UserCommandStatistics stats = new UserCommandStatistics();
- stats.setCommandName(this.cmdTypes[i]);
- RealVector colData = m.getColumnVector(i);
- StandardDeviation deviation = new StandardDeviation();
- double stddev = deviation.evaluate(colData.toArray());
- //LOG.info("stddev is nan?" + (stddev == Double.NaN? "yes":"no"));
- if(stddev <= lowVarianceVal)
- stats.setLowVariant(true);
- else
- stats.setLowVariant(false);
- stats.setStddev(stddev);
- Mean mean = new Mean();
- double mu = mean.evaluate(colData.toArray());
- //LOG.info("mu is nan?" + (mu == Double.NaN? "yes":"no"));
- stats.setMean(mu);
- statistics[i] = stats;
- }
- }
-
- private RealMatrix normalizeData(RealMatrix matrix){
-
- RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
- for(int i=0; i< matrix.getRowDimension(); i++){
- for(int j=0; j < matrix.getColumnDimension(); j++){
- // TODO: statistics[j].getStddev() == 0 what should the value be if stddev is o?
- double value = statistics[j].getStddev() == 0? 0 : (matrix.getEntry(i, j) - statistics[j].getMean())/statistics[j].getStddev();
- normalizedData.setEntry(i, j, value);
- }
- }
- return normalizedData;
- }
-
- private void computeCovarianceAndSVD(RealMatrix inputMat, int containsLowVariantCol){
-
- int finalMatrixRow=0;
- int finalMatrixCol=0;
-
- LOG.info("containsLowVariantCol size: " + containsLowVariantCol);
- int colDimension = (inputMat.getColumnDimension() - containsLowVariantCol);
- try {
- finalMatrixWithoutLowVariantCmds = new Array2DRowRealMatrix(inputMat.getRowDimension(), colDimension);
- }catch (NotStrictlyPositiveException e){
- LOG.error(String.format("Failed to build matrix [rowDimension:%s, columnDimension: %s]",inputMat.getRowDimension(),colDimension),e);
- throw e;
- }
-
- for(int i=0; i < inputMat.getRowDimension(); i++){
- for(int j=0; j < inputMat.getColumnDimension();j++){
- if(!statistics[j].isLowVariant()){
- finalMatrixWithoutLowVariantCmds.setEntry(finalMatrixRow, finalMatrixCol, inputMat.getEntry(i, j));
- finalMatrixCol++;
- }
- }
- finalMatrixCol = 0;
- finalMatrixRow++;
- }
-
- Covariance cov;
- try {
- cov = new Covariance(finalMatrixWithoutLowVariantCmds.getData());
- }catch (Exception ex){
- throw new IllegalArgumentException(String.format("Failed to create covariance from matrix [ %s x %s ]",finalMatrixWithoutLowVariantCmds.getRowDimension(),finalMatrixWithoutLowVariantCmds.getColumnDimension()),ex);
- }
- covarianceMatrix = cov.getCovarianceMatrix();
- SingularValueDecomposition svd = new SingularValueDecomposition(covarianceMatrix);
- diagonalMatrix = svd.getS();
- uMatrix = svd.getU();
- vMatrix = svd.getV();
- }
-
- private void computeDimensionWithMaxVariance(){
- if(diagonalMatrix == null || diagonalMatrix.getRowDimension() != diagonalMatrix.getColumnDimension()){
- LOG.error("Diagonal matrix is not correctly computed, SVD should have been done");
- return;
- }
-
- double sumTotal = 0.0;
- for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
- sumTotal += diagonalMatrix.getEntry(i, i);
- }
- //LOG.info("sum total: " + sumTotal);
- double sumEach = 0.0;
- for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
- sumEach += diagonalMatrix.getEntry(i, i);
- if((sumEach/sumTotal) >= MAINTENED_VARIANCE){
- dimension = i;
- break;
- }
-
- }
- if(dimension > finalMatrixWithoutLowVariantCmds.getColumnDimension())
- LOG.error("Reduced dimension cannot be smaller than original size of the feature");
-
- }
-
- private void computePrincipalComponents(){
- principalComponents = new ArrayRealVector[uMatrix.getColumnDimension()];
- for(int i=0; i < uMatrix.getColumnDimension(); i++){
- principalComponents[i] = uMatrix.getColumnVector(i).mapMultiply(Math.sqrt(diagonalMatrix.getEntry(i, i)));
- }
- }
-
- private RealMatrix computeMaxDistanceOnPCs(int index){
-
- RealMatrix pc = new Array2DRowRealMatrix(principalComponents[index].toArray());
- RealMatrix transposePC1 = pc.transpose();
- RealMatrix finalDataTranspose = finalMatrixWithoutLowVariantCmds.transpose();
-
- RealMatrix trainingData = pc.multiply(transposePC1).multiply(finalDataTranspose);
- RealMatrix trainingDataTranspose = trainingData.transpose();
-
- double maxDistance = 0.0;
- double minDistance = 0.0;
- RealVector p1 = null, p2 = null;
- if(LOG.isDebugEnabled()) LOG.debug("Training data transpose size: " + trainingDataTranspose.getRowDimension() + "x" + trainingDataTranspose.getColumnDimension());
- for(int i = 0; i < trainingDataTranspose.getRowDimension(); i++){
- RealVector iRowVector = new ArrayRealVector(trainingDataTranspose.getRow(i));
- RealVector transposePC1Vect = transposePC1.getRowVector(0);
- double distance = iRowVector.getDistance(transposePC1Vect);
- if(distance > maxDistance){
- maxDistance = distance;
- p1 = iRowVector;
- p2 = transposePC1Vect;
- }
- if(distance < minDistance)
- minDistance = distance;
- //}
- }
- maximumL2Norm.setEntry(index, maxDistance);
- minimumL2Norm.setEntry(index, minDistance);
- minVector = p1;
- maxVector = p2;
- return trainingDataTranspose;
- }
-
- @Override
- public List<UserProfileEigenModel> generate(String site, String user, RealMatrix matrix) {
- LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
- computeStats(matrix);
- RealMatrix normalizedInputMatrix = normalizeData(matrix);
- int lowVariantColumnCount = 0;
- for(int j=0; j < normalizedInputMatrix.getColumnDimension();j++){
- if(statistics[j].isLowVariant()){
- lowVariantColumnCount++;
- }
- }
-
- if(normalizedInputMatrix.getColumnDimension() == lowVariantColumnCount){
- LOG.info("found user: " + user + " with all features being low variant. Nothing to do...");
- UserProfileEigenModel noopModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user,null,null,0,null,null,null,null,null,statistics);
- return Arrays.asList(noopModel);
- }
- else {
- computeCovarianceAndSVD(normalizedInputMatrix, lowVariantColumnCount);
- computeDimensionWithMaxVariance();
- computePrincipalComponents();
- maximumL2Norm = new ArrayRealVector(principalComponents.length);
- minimumL2Norm = new ArrayRealVector(principalComponents.length);
-
- for (int i = 0; i < principalComponents.length; i++) {
- RealMatrix trainingDataTranspose = computeMaxDistanceOnPCs(i);
- }
-
- UserProfileEigenModel userprofileEigenModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user, uMatrix, diagonalMatrix, dimension, minVector, maxVector, principalComponents, maximumL2Norm, minimumL2Norm, statistics);
- return Arrays.asList(userprofileEigenModel);
- }
- }
-
- @Override
- public UserProfileContext context() {
- return new UserProfileContext(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
deleted file mode 100644
index 1b24137..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.kde;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.JavaUserProfileModeler;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileContext;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class UserProfileKDEModeler extends JavaUserProfileModeler<UserProfileKDEModel,UserProfileContext> {
-
- private final String[] cmdTypes;
- private UserCommandStatistics[] statistics;
- private RealMatrix finalMatrixWithoutLowVariantCmds;
- private RealMatrix covarianceMatrix;
- private double[] probabilityEstimation;
- private double minProbabilityEstimate;
- private double maxProbabilityEstimate;
- private double nintyFivePercentileEstimate;
- private double medianProbabilityEstimate;
- private final static Logger LOG = LoggerFactory.getLogger(UserProfileKDEModeler.class);
- private static final double lowVarianceVal= 0.001;
-
- public UserProfileKDEModeler(String[] cmdTypes){
- this.cmdTypes = cmdTypes;
- }
-
- public UserProfileKDEModeler(){
- this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
- }
-
- private void computeStats(RealMatrix m){
- if(m.getColumnDimension() != this.cmdTypes.length){
- LOG.error("Please fix the commands list in config file");
- }
-
- statistics = new UserCommandStatistics[m.getColumnDimension()];
-
- for(int i=0; i<m.getColumnDimension(); i++){
- UserCommandStatistics stats = new UserCommandStatistics();
- stats.setCommandName(this.cmdTypes[i]);
- RealVector colData = m.getColumnVector(i);
- StandardDeviation deviation = new StandardDeviation();
- double stddev = deviation.evaluate(colData.toArray());
-
- if(LOG.isDebugEnabled()) LOG.debug("Stddev is NAN ? " + (Double.isNaN(stddev) ? "yes" : "no"));
- if(stddev <= lowVarianceVal)
- stats.setLowVariant(true);
- else
- stats.setLowVariant(false);
-
- stats.setStddev(stddev);
- Mean mean = new Mean();
- double mu = mean.evaluate(colData.toArray());
- if(LOG.isDebugEnabled()) LOG.debug("mu is NAN ? " + (Double.isNaN(mu)? "yes":"no"));
-
- stats.setMean(mu);
- statistics[i]=stats;
- }
- }
-
- private void computeProbabilityDensityEstimation(RealMatrix inputMat){
-
- probabilityEstimation = new double[inputMat.getRowDimension()];
- for(int i=0; i < probabilityEstimation.length; i++)
- probabilityEstimation[i] = 1.0;
-
- for(int i=0; i < inputMat.getRowDimension(); i++){
- for(int j=0; j < inputMat.getColumnDimension(); j++){
- if(statistics[j].getStddev() > 0){
- double stddev = statistics[j].getStddev();
- double mean = statistics[j].getMean();
- double sqrt2PI = Math.sqrt(2.0*Math.PI);
- double denominatorFirstPart = sqrt2PI*stddev;
- double squareMeanNormal = Math.pow((inputMat.getEntry(i, j) - mean), 2);
- double twoPowStandardDev = Math.pow(stddev, 2);
- double twoTimesTwoPowStandardDev = 2.0*twoPowStandardDev;
- probabilityEstimation[i] *= ((1.00/denominatorFirstPart)
- *(Math.exp(-(squareMeanNormal/twoTimesTwoPowStandardDev))));
- }
- }
- }
-
- java.util.List<Double> listProb = new ArrayList<Double>();
- for(int i=0; i < probabilityEstimation.length; i++){
- probabilityEstimation[i] = Math.log10(probabilityEstimation[i]);
- listProb.add(probabilityEstimation[i]);
- }
-
- Collections.sort(listProb);
- int i=0;
- for(double d:listProb)
- probabilityEstimation[i++]=d;
-
- minProbabilityEstimate = probabilityEstimation[probabilityEstimation.length -1];
- maxProbabilityEstimate = probabilityEstimation[0];
-
- int len = probabilityEstimation.length;
- int nintyFivePercentIndex = (int)Math.round(0.05*len);
- int medianPercentIndex = (int)Math.round(0.5*len);
- if(medianPercentIndex >= len)
- medianProbabilityEstimate = probabilityEstimation[medianPercentIndex-1];
- else
- medianProbabilityEstimate = probabilityEstimation[medianPercentIndex];
- nintyFivePercentileEstimate = probabilityEstimation[nintyFivePercentIndex];
- }
-
- @Override
- public List<UserProfileKDEModel> generate(String site, String user, RealMatrix matrix) {
- LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
- computeStats(matrix);
- computeProbabilityDensityEstimation(matrix);
- UserProfileKDEModel userprofileKDEModel = new UserProfileKDEModel(System.currentTimeMillis(),site,user, statistics, minProbabilityEstimate, maxProbabilityEstimate, nintyFivePercentileEstimate, medianProbabilityEstimate);
- return Arrays.asList(userprofileKDEModel);
- }
-
- @Override
- public UserProfileContext context() {
- return new UserProfileContext(UserProfileConstants.KDE_ALGORITHM);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf b/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
deleted file mode 100644
index db896d0..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-eagle{
- site = "sandbox"
-
- service{
- host = "localhost"
- port = 9098
- username = "admin"
- password = "secret"
- }
-
- userprofile {
- jar="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/eagle-security-userprofile-training-0.1.0-assembly.jar"
- driver-shell="spark-submit"
- driver-classpath="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/lib/commons-math3-3.5.jar"
- // model-audit-path = "/tmp/audit/${yyyy-MM-*}/*.tar.gz"
- // agg-audit-path = "/tmp/audit/${yyyy-MM-dd}/${yyyy-MM-dd-hh}.tar.gz"
-
- training-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
- detection-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
- }
-}
\ No newline at end of file