You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/11/19 11:47:40 UTC
[33/55] [abbrv] [partial] incubator-eagle git commit: [EAGLE-46]
Rename package name as "org.apache.eagle"
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLAnomalyCallback.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLAnomalyCallback.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLAnomalyCallback.java
deleted file mode 100644
index 481ebfa..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLAnomalyCallback.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml;
-
-import eagle.alert.siddhi.EagleAlertContext;
-import eagle.ml.model.MLCallbackResult;
-
-public interface MLAnomalyCallback {
- /**
- * @param callbackResult call-backed result
- * @param alertContext context
- */
- void receive(MLCallbackResult callbackResult,EagleAlertContext alertContext);
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLConstants.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLConstants.java
deleted file mode 100644
index 1ff3f76..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLConstants.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package eagle.ml;
-
-public class MLConstants {
- public final static String ML_MODEL_SERVICE_NAME = "MLModelService";
- public final static String ALGORITHM = "algorithm";
- public final static String USER = "user";
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLModelDAO.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLModelDAO.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLModelDAO.java
deleted file mode 100644
index 011adb5..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLModelDAO.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml;
-
-import eagle.ml.model.MLModelAPIEntity;
-
-import java.util.List;
-
-public interface MLModelDAO {
- /**
- * @param user
- * @param algorithm
- * @return
- */
- List<MLModelAPIEntity> findMLModelByContext(String user, String algorithm);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLPolicyEvaluator.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLPolicyEvaluator.java
deleted file mode 100644
index cecf64d..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/MLPolicyEvaluator.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml;
-
-import eagle.alert.common.AlertConstants;
-import eagle.alert.config.AbstractPolicyDefinition;
-import eagle.alert.entity.AlertDefinitionAPIEntity;
-import eagle.alert.policy.PolicyEvaluator;
-import eagle.alert.policy.PolicyManager;
-import eagle.dataproc.core.JsonSerDeserUtils;
-import eagle.dataproc.core.ValuesArray;
-import eagle.ml.impl.MLAnomalyCallbackImpl;
-import eagle.ml.model.MLAlgorithm;
-import eagle.ml.model.MLPolicyDefinition;
-import eagle.ml.utils.MLReflectionUtils;
-import com.typesafe.config.Config;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-public class MLPolicyEvaluator implements PolicyEvaluator {
- private static Logger LOG = LoggerFactory.getLogger(MLPolicyEvaluator.class);
- private final String[] sourceStreams;
- private volatile MLRuntime mlRuntime;
- private String policyName;
- private Config config;
- private Map<String,String> context;
-
- private class MLRuntime{
- MLPolicyDefinition mlPolicyDef;
- MLAlgorithmEvaluator[] mlAlgorithmEvaluators;
- List<MLAnomalyCallback> mlAnomalyCallbacks = new ArrayList<>();
- }
-
- public MLPolicyEvaluator(Config config, String policyName, AbstractPolicyDefinition policyDef, String[] sourceStreams){
- this(config, policyName, policyDef, sourceStreams, false);
- }
-
- /**
- * needValidation does not take effect for machine learning use case
- * @param policyDef
- * @param sourceStreams
- * @param needValidation
- */
- public MLPolicyEvaluator(Config config, String policyName, AbstractPolicyDefinition policyDef, String[] sourceStreams, boolean needValidation){
- this.config = config;
- this.policyName = policyName;
- this.sourceStreams = sourceStreams;
- LOG.info("Initializing policy named: "+policyName);
- this.context = new HashMap<>();
- this.context.put(AlertConstants.SOURCE_STREAMS, StringUtils.join(sourceStreams,","));
- this.init(policyDef);
- }
-
- public void init(AbstractPolicyDefinition policyDef){
- LOG.info("Initializing MLPolicyEvaluator ...");
- try{
- mlRuntime = newMLRuntime((MLPolicyDefinition) policyDef);
- }catch(Exception e){
- LOG.error("ML Runtime creation failed: " + e.getMessage());
- }
- }
-
- private MLRuntime newMLRuntime(MLPolicyDefinition mlPolicyDef) {
- MLRuntime runtime = new MLRuntime();
-
- try{
- runtime.mlPolicyDef = mlPolicyDef;
-
- LOG.info("policydef: " + ((runtime.mlPolicyDef == null)? "policy definition is null": "policy definition is not null"));
- Properties alertContext = runtime.mlPolicyDef.getContext();
- LOG.info("alert context received null? " + ((alertContext == null? "yes": "no")));
- MLAnomalyCallback callbackImpl = new MLAnomalyCallbackImpl(this, config);
- runtime.mlAnomalyCallbacks.add(callbackImpl);
-
- MLAlgorithm[] mlAlgorithms = mlPolicyDef.getAlgorithms();
- runtime.mlAlgorithmEvaluators = new MLAlgorithmEvaluator[mlAlgorithms.length];
- LOG.info("mlAlgorithms size:: " + mlAlgorithms.length);
- int i = 0;
- for(MLAlgorithm algorithm:mlAlgorithms){
- MLAlgorithmEvaluator mlAlgorithmEvaluator = MLReflectionUtils.newMLAlgorithmEvaluator(algorithm);
- mlAlgorithmEvaluator.init(algorithm,config);
- runtime.mlAlgorithmEvaluators[i] = mlAlgorithmEvaluator;
- LOG.info("mlAlgorithmEvaluator: " + mlAlgorithmEvaluator.toString());
- mlAlgorithmEvaluator.register(callbackImpl);
- i++;
- }
- }catch(Exception ex){
- LOG.error("Failed to create runtime for policy named: "+this.getPolicyName(),ex);
- }
- return runtime;
- }
-
- @Override
- public void evaluate(ValuesArray data) throws Exception {
- LOG.info("Evaluate called with input: " + data.size());
- synchronized(mlRuntime){
- for(MLAlgorithmEvaluator mlAlgorithm:mlRuntime.mlAlgorithmEvaluators){
- mlAlgorithm.evaluate(data);
- }
- }
- }
-
- @Override
- public void onPolicyUpdate(AlertDefinitionAPIEntity newAlertDef) {
- LOG.info("onPolicyUpdate called");
- AbstractPolicyDefinition policyDef = null;
- try {
- policyDef = JsonSerDeserUtils.deserialize(newAlertDef.getPolicyDef(),
- AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(newAlertDef.getTags().get("policyType")));
- } catch (Exception ex) {
- LOG.error("initial policy def error, ", ex);
- }
- MLRuntime previous = mlRuntime;
- mlRuntime = newMLRuntime((MLPolicyDefinition) policyDef);
- synchronized (previous) {
- previous.mlAnomalyCallbacks = null;
- previous.mlAlgorithmEvaluators = null;
- previous.mlPolicyDef = null;
- }
- previous = null;
- }
-
- @Override
- public void onPolicyDelete() {
- LOG.info("onPolicyDelete called");
- MLRuntime previous = mlRuntime;
- synchronized (previous) {
- previous.mlAnomalyCallbacks = null;
- previous.mlAlgorithmEvaluators = null;
- previous.mlPolicyDef = null;
- }
- previous = null;
- }
-
- public String getPolicyName() {
- return policyName;
- }
-
- public Map<String, String> getAdditionalContext() {
- return this.context;
- }
-
- public List<String> getOutputStreamAttrNameList() {
- return new ArrayList<String>();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLAnomalyCallbackImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLAnomalyCallbackImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLAnomalyCallbackImpl.java
deleted file mode 100644
index 3af8c0f..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLAnomalyCallbackImpl.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.impl;
-
-import eagle.alert.common.AlertConstants;
-import eagle.alert.entity.AlertAPIEntity;
-import eagle.alert.siddhi.EagleAlertContext;
-import eagle.common.DateTimeUtil;
-import eagle.common.config.EagleConfigConstants;
-import eagle.ml.MLAnomalyCallback;
-import eagle.ml.MLPolicyEvaluator;
-import eagle.ml.model.MLCallbackResult;
-import eagle.common.metric.AlertContext;
-import com.typesafe.config.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-public class MLAnomalyCallbackImpl implements MLAnomalyCallback {
- private static Logger LOG = LoggerFactory.getLogger(MLAnomalyCallbackImpl.class);
- private MLPolicyEvaluator mlAlertEvaluator;
- private Config config;
-
-
- public static final String source = ManagementFactory.getRuntimeMXBean().getName();
-
- public MLAnomalyCallbackImpl(MLPolicyEvaluator mlAlertEvaluator, Config config){
- this.mlAlertEvaluator = mlAlertEvaluator;
- this.config = config;
- }
-
- /**
- * TODO: generate alert
- *
- * @param aResult
- * @param alertContext context
- */
- @Override
- public void receive(MLCallbackResult aResult,EagleAlertContext alertContext) {
- LOG.info("Receive called with : " + aResult.toString());
- AlertAPIEntity alert = renderAlert(aResult,alertContext);
- alertContext.alertExecutor.onAlerts(alertContext, Arrays.asList(alert));
- }
-
- private AlertAPIEntity renderAlert(MLCallbackResult aResult,EagleAlertContext alertContext){
- String site = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.SITE);
- String dataSource = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.DATA_SOURCE);
-
- AlertAPIEntity entity = new AlertAPIEntity();
- entity.setDescription(aResult.toString());
-
- Map<String, String> tags = new HashMap<>();
- tags.put(EagleConfigConstants.SITE, site);
- tags.put(EagleConfigConstants.DATA_SOURCE, dataSource);
- tags.put(AlertConstants.SOURCE_STREAMS, alertContext.evaluator.getAdditionalContext().get(AlertConstants.SOURCE_STREAMS));
- tags.put(AlertConstants.POLICY_ID, alertContext.policyId);
- tags.put(AlertConstants.ALERT_SOURCE, source);
- tags.put(AlertConstants.ALERT_EXECUTOR_ID, alertContext.alertExecutor.getAlertExecutorId());
- entity.setTags(tags);
-
- entity.setTimestamp(aResult.getTimestamp());
-
- AlertContext context = new AlertContext();
-
- if(aResult.getContext() != null) context.addAll(aResult.getContext());
-
- String alertMessage = "Anomaly activities detected by algorithm ["+aResult.getAlgorithmName()+"] with information: " + aResult.toString() ;
- context.addProperty(AlertConstants.ALERT_EVENT, aResult.toString());
- context.addProperty(AlertConstants.ALERT_MESSAGE, alertMessage);
- context.addProperty(AlertConstants.ALERT_TIMESTAMP_PROPERTY, DateTimeUtil.millisecondsToHumanDateWithSeconds(System.currentTimeMillis()));
-
- try {
- site = config.getString("eagleProps.site");
- dataSource = config.getString("eagleProps.dataSource");
- context.addProperty(EagleConfigConstants.DATA_SOURCE, dataSource);
- context.addProperty(EagleConfigConstants.SITE, site);
- } catch (Exception ex) {
- LOG.error("site, dataSource not set in config file, ", ex);
- }
-
- context.addProperty(EagleConfigConstants.DATA_SOURCE, dataSource);
- context.addProperty(EagleConfigConstants.SITE, site);
- context.addProperty(AlertConstants.POLICY_NAME, alertContext.policyId);
-
- entity.setAlertContext(context);
- return entity;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLModelDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLModelDAOImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLModelDAOImpl.java
deleted file mode 100644
index 8ab175f..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLModelDAOImpl.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.impl;
-
-import com.typesafe.config.Config;
-import eagle.alert.dao.AlertStreamSchemaDAOImpl;
-import eagle.common.config.EagleConfigConstants;
-import eagle.log.entity.GenericServiceAPIResponseEntity;
-import eagle.ml.MLConstants;
-import eagle.ml.MLModelDAO;
-import eagle.ml.model.MLModelAPIEntity;
-import eagle.service.client.IEagleServiceClient;
-import eagle.service.client.impl.EagleServiceClientImpl;
-import org.apache.commons.lang.time.DateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public class MLModelDAOImpl implements MLModelDAO {
- private final Logger LOG = LoggerFactory.getLogger(AlertStreamSchemaDAOImpl.class);
-
- private String eagleServiceHost;
- private int eagleServicePort;
- private String eagleServiceUserName;
- private String eagleServicePassword;
-
- public MLModelDAOImpl(String eagleServiceHost, int eagleServicePort){
- this.eagleServiceHost = eagleServiceHost;
- this.eagleServicePort = eagleServicePort;
- }
-
- public MLModelDAOImpl(String eagleServiceHost, int eagleServicePort, String eagleServiceUserName, String eagleServicePassword){
- this.eagleServiceHost = eagleServiceHost;
- this.eagleServicePort = eagleServicePort;
- this.eagleServiceUserName = eagleServiceUserName;
- this.eagleServicePassword = eagleServicePassword;
- }
-
- public MLModelDAOImpl(Config config){
- this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
- this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
- this.eagleServiceUserName =config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
- this.eagleServicePassword = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD);
- }
-
- @Override
- public List<MLModelAPIEntity> findMLModelByContext(String user, String algorithm) {
- try {
- IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, eagleServiceUserName, eagleServicePassword);
- String query = MLConstants.ML_MODEL_SERVICE_NAME + "[@user=\"" + user + "\" AND @algorithm=\""
- + algorithm + "\"]{*}";
- GenericServiceAPIResponseEntity<MLModelAPIEntity> response = client.search().startTime(0)
- .endTime(10 * DateUtils.MILLIS_PER_DAY)
- .pageSize(Integer.MAX_VALUE)
- .query(query)
- .send();
- if(!response.isSuccess()) {
- LOG.error(String.format("Failed to get model for user: %s, algorithm: %s, due to: %s",user,algorithm,response.getException()));
- }
-
- client.close();
- return response.getObj();
- } catch (Exception ex) {
- LOG.info("Got an exception when query machinelearning model service ", ex);
- throw new IllegalStateException(ex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
deleted file mode 100644
index fb289b8..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.impl;
-
-import eagle.alert.common.AlertConstants;
-import eagle.alert.policy.PolicyEvaluator;
-import eagle.alert.policy.PolicyEvaluatorServiceProvider;
-import eagle.ml.MLPolicyEvaluator;
-import eagle.ml.model.MLPolicyDefinition;
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-public class MLPolicyEvaluatorServiceProviderImpl implements PolicyEvaluatorServiceProvider {
-
- private final static String ALERT_CONTEXT = "alertContext";
-
- @Override
- public String getPolicyType() {
- return AlertConstants.policyType.MachineLearning.name();
- }
-
- @Override
- public Class<? extends PolicyEvaluator> getPolicyEvaluator() {
- return MLPolicyEvaluator.class;
- }
-
- @Override
- public List<Module> getBindingModules() {
- Module module1 = new SimpleModule(AlertConstants.POLICY_DEFINITION).registerSubtypes(new NamedType(MLPolicyDefinition.class, getPolicyType()));
- Module module2 = new SimpleModule(ALERT_CONTEXT).registerSubtypes(new NamedType(Properties.class, getPolicyType()));
- return Arrays.asList(module1, module2);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLAlgorithm.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLAlgorithm.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLAlgorithm.java
deleted file mode 100644
index f4d0241..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLAlgorithm.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.model;
-
-import java.io.Serializable;
-
-public class MLAlgorithm implements Serializable {
- private static final long serialVersionUID = -223531147520123L;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- private String name;
- private String evaluator;
- private String description;
-
- public String getFeatures() {
- return features;
- }
-
- public void setFeatures(String features) {
- this.features = features;
- }
-
- private String features;
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public String getEvaluator() {
- return evaluator;
- }
-
- public void setEvaluator(String evaluator) {
- this.evaluator = evaluator;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLCallbackResult.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLCallbackResult.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLCallbackResult.java
deleted file mode 100644
index 0e9c6a3..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLCallbackResult.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.model;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class MLCallbackResult {
- private boolean isAnomaly;
- private List<String> feature;
- private double confidence;
- private long timestamp;
- private List<String> datapoints;
- private String id;
- private String algorithmName;
-
- public Map<String, String> getContext() {
- return context;
- }
-
- public void setContext(Map<String, String> context) {
- this.context = context;
- }
-
- public void setAlgorithmName(String algorithmName) {
- this.algorithmName = algorithmName;
- }
-
- private Map<String,String> context;
-
- public String getAlgorithmName() {
- return algorithmName;
- }
- public void setAlgorithm(String algorithmName) {
- this.algorithmName = algorithmName;
- }
- public MLCallbackResult(){
- feature = new ArrayList<String>();
- setDatapoints(new ArrayList<String>());
- }
- public boolean isAnomaly() {
- return isAnomaly;
- }
- public void setAnomaly(boolean isAnomaly) {
- this.isAnomaly = isAnomaly;
- }
- public List<String> getFeature() {
- return feature;
- }
- public void setFeature(List<String> feature) {
- this.feature = feature;
- }
-
- private boolean doesFeatureExist(String f){
- boolean alreadyExist = false;
- for(String s:feature){
- if(s.equalsIgnoreCase(f))
- alreadyExist = true;
- }
- return alreadyExist;
- }
- public void setFeature(String aFeature) {
- if(doesFeatureExist(aFeature) == false)
- feature.add(aFeature);
- }
- public double getConfidence() {
- return confidence;
- }
- public void setConfidence(double confidence) {
- this.confidence = confidence;
- }
- public String toString(){
- StringBuffer resultStr = new StringBuffer();
- resultStr.append("datapoint :<");
- int i=0;
- for(String d:datapoints){
- if(i < datapoints.size()-1){
- resultStr.append(d.trim());
- resultStr.append(",");
- i++;
- }else
- resultStr.append(d.trim());
- }
- resultStr.append("> for user: ").append(id).append(" is anomaly: ").append(this.isAnomaly).append(" at timestamp: ").append(this.timestamp);
- if(this.isAnomaly){
- //resultStr.append(" with confidence: " + this.confidence);
- resultStr.append(" with algorithm: " + algorithmName);
- resultStr.append(" with features: ");
- int p=0;
- resultStr.append("[");
- for(String s:feature){
- if(p < feature.size()-1){
- resultStr.append(s.trim());
- resultStr.append(",");
- p++;
- }else
- resultStr.append(s.trim());
- }
- resultStr.append("]");
- }
- return resultStr.toString();
- }
-
- public long getTimestamp() {
- return timestamp;
- }
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
- public List<String> getDatapoints() {
- return datapoints;
- }
- public void setDatapoints(List<String> datapoints) {
- this.datapoints = datapoints;
- }
- public String getId() {
- return id;
- }
- public void setId(String id) {
- this.id = id;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLEntityRepository.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLEntityRepository.java
deleted file mode 100644
index 4f7139f..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLEntityRepository.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.model;
-
-import eagle.log.entity.repo.EntityRepository;
-
-public class MLEntityRepository extends EntityRepository {
- public MLEntityRepository() {
- entitySet.add(MLModelAPIEntity.class);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLModelAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLModelAPIEntity.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLModelAPIEntity.java
deleted file mode 100644
index a9e7495..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLModelAPIEntity.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.model;
-
-import eagle.ml.MLConstants;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import eagle.log.entity.meta.Column;
-import eagle.log.entity.meta.ColumnFamily;
-import eagle.log.entity.meta.Prefix;
-import eagle.log.entity.meta.Service;
-import eagle.log.entity.meta.Table;
-import eagle.log.entity.meta.Tags;
-import eagle.log.entity.meta.TimeSeries;
-
-/**
- * DDL for creating the table
- * create 'mlmodel', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
- */
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
-@Table("mlmodel")
-@ColumnFamily("f")
-@Prefix("mlmodel")
-@Service(MLConstants.ML_MODEL_SERVICE_NAME)
-@JsonIgnoreProperties(ignoreUnknown = true)
-@TimeSeries(false)
-@Tags({"site","user", "algorithm"})
-public class MLModelAPIEntity extends TaggedLogAPIEntity{
- @Column("b")
- private String content;
- @Column("c")
- private long version;
-
- public String getContent() {
- return content;
- }
-
- public void setContent(String content) {
- this.content = content;
- valueChanged("content");
- }
-
- public long getVersion() {
- return version;
- }
-
- public void setVersion(long version) {
- this.version = version;
- valueChanged("version");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLPolicyDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLPolicyDefinition.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLPolicyDefinition.java
deleted file mode 100644
index 70e312f..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/model/MLPolicyDefinition.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.model;
-
-import eagle.alert.config.AbstractPolicyDefinition;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-import java.util.Properties;
-
-/**
- {
- "type":"MachineLearning",
- "context":{
- "site":"dev",
- "dataSource":"userprofile"
- "component":"dev-component",
- "description":"ML based user profile anomaly detection",
- "severity":"WARNING",
- "notificationByEmail":"true"
- },
- "algorithms":[
- {
- "name":"eagle.security.userprofile.util.EigenBasedAnomalyDetection",
- "description":"EigenBasedAnomalyDetection",
- "features":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete"
- },
- {
- "name":"eagle.security.userprofile.util.KDEBasedAnomalyDetection",
- "description":"DensityBasedAnomalyDetection",
- "features":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete"
- }
- ]
-}
- version field is used for model update, so eagle framework can understand that something changes.
-
- */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible=true)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class MLPolicyDefinition extends AbstractPolicyDefinition{
- private String version;
- private Properties context;
- private MLAlgorithm[] algorithms;
-
- public MLAlgorithm[] getAlgorithms() {
- return algorithms;
- }
-
- public void setAlgorithms(MLAlgorithm[] algorithms) {
- this.algorithms = algorithms;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public Properties getContext() {
- return context;
- }
-
- public void setContext(Properties context) {
- this.context = context;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/utils/MLReflectionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/utils/MLReflectionUtils.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/utils/MLReflectionUtils.java
deleted file mode 100644
index 41ed3da..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/eagle/ml/utils/MLReflectionUtils.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package eagle.ml.utils;
-
-import eagle.ml.MLAlgorithmEvaluator;
-import eagle.ml.model.MLAlgorithm;
-
-/**
- * @since 8/14/15
- */
-public class MLReflectionUtils {
- /**
- * Create a new MLAlgorithmEvaluator based on MLAlgorithm
- *
- * @param algorithm MLAlgorithm
- * @return MLAlgorithmEvaluator object instance
- * @throws ClassNotFoundException
- * @throws IllegalAccessException
- * @throws InstantiationException
- */
- public static MLAlgorithmEvaluator newMLAlgorithmEvaluator(MLAlgorithm algorithm) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
- return (MLAlgorithmEvaluator) Class.forName(algorithm.getEvaluator()).newInstance();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAlgorithmEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAlgorithmEvaluator.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAlgorithmEvaluator.java
new file mode 100644
index 0000000..4ca5efe
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAlgorithmEvaluator.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml;
+
+import org.apache.eagle.dataproc.core.ValuesArray;
+import org.apache.eagle.ml.model.MLAlgorithm;
+import com.typesafe.config.Config;
+
+/**
+ * Machine Learning Algorithm Evaluator
+ */
+public interface MLAlgorithmEvaluator {
+ /**
+ * Prepare Machine learning algorithm
+ *
+ * @param algorithm MLAlgorithm instance
+ */
+ public void init(MLAlgorithm algorithm, Config config);
+
+ /**
+ * Evaluate input user profile model
+ *
+ * @param data ValuesArray
+ * @throws Exception
+ */
+ public void evaluate(ValuesArray data) throws Exception;
+
+ /**
+ * Register callback
+ *
+ * @param callbackObj MachineLearningCallback
+ * @throws Exception
+ */
+ public void register(MLAnomalyCallback callbackObj) throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAnomalyCallback.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAnomalyCallback.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAnomalyCallback.java
new file mode 100644
index 0000000..22b3e5c
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLAnomalyCallback.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml;
+
+import org.apache.eagle.alert.siddhi.EagleAlertContext;
+import org.apache.eagle.ml.model.MLCallbackResult;
+
+public interface MLAnomalyCallback {
+ /**
+ * @param callbackResult call-backed result
+ * @param alertContext context
+ */
+ void receive(MLCallbackResult callbackResult,EagleAlertContext alertContext);
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLConstants.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLConstants.java
new file mode 100644
index 0000000..9197d59
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLConstants.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.ml;
+
+public class MLConstants {
+ public final static String ML_MODEL_SERVICE_NAME = "MLModelService";
+ public final static String ALGORITHM = "algorithm";
+ public final static String USER = "user";
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLModelDAO.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLModelDAO.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLModelDAO.java
new file mode 100644
index 0000000..13eb7e1
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLModelDAO.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml;
+
+import org.apache.eagle.ml.model.MLModelAPIEntity;
+
+import java.util.List;
+
+public interface MLModelDAO {
+ /**
+ * @param user
+ * @param algorithm
+ * @return
+ */
+ List<MLModelAPIEntity> findMLModelByContext(String user, String algorithm);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLPolicyEvaluator.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLPolicyEvaluator.java
new file mode 100644
index 0000000..c2a8d6b
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/MLPolicyEvaluator.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml;
+
+import org.apache.eagle.alert.common.AlertConstants;
+import org.apache.eagle.alert.config.AbstractPolicyDefinition;
+import org.apache.eagle.alert.entity.AlertDefinitionAPIEntity;
+import org.apache.eagle.alert.policy.PolicyEvaluator;
+import org.apache.eagle.alert.policy.PolicyManager;
+import org.apache.eagle.dataproc.core.JsonSerDeserUtils;
+import org.apache.eagle.dataproc.core.ValuesArray;
+import org.apache.eagle.ml.impl.MLAnomalyCallbackImpl;
+import org.apache.eagle.ml.model.MLAlgorithm;
+import org.apache.eagle.ml.model.MLPolicyDefinition;
+import org.apache.eagle.ml.utils.MLReflectionUtils;
+import com.typesafe.config.Config;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class MLPolicyEvaluator implements PolicyEvaluator {
+ private static Logger LOG = LoggerFactory.getLogger(MLPolicyEvaluator.class);
+ private final String[] sourceStreams;
+ private volatile MLRuntime mlRuntime;
+ private String policyName;
+ private Config config;
+ private Map<String,String> context;
+
+ private class MLRuntime{
+ MLPolicyDefinition mlPolicyDef;
+ MLAlgorithmEvaluator[] mlAlgorithmEvaluators;
+ List<MLAnomalyCallback> mlAnomalyCallbacks = new ArrayList<>();
+ }
+
+ public MLPolicyEvaluator(Config config, String policyName, AbstractPolicyDefinition policyDef, String[] sourceStreams){
+ this(config, policyName, policyDef, sourceStreams, false);
+ }
+
+ /**
+ * needValidation does not take effect for machine learning use case
+ * @param policyDef
+ * @param sourceStreams
+ * @param needValidation
+ */
+ public MLPolicyEvaluator(Config config, String policyName, AbstractPolicyDefinition policyDef, String[] sourceStreams, boolean needValidation){
+ this.config = config;
+ this.policyName = policyName;
+ this.sourceStreams = sourceStreams;
+ LOG.info("Initializing policy named: "+policyName);
+ this.context = new HashMap<>();
+ this.context.put(AlertConstants.SOURCE_STREAMS, StringUtils.join(sourceStreams,","));
+ this.init(policyDef);
+ }
+
+ public void init(AbstractPolicyDefinition policyDef){
+ LOG.info("Initializing MLPolicyEvaluator ...");
+ try{
+ mlRuntime = newMLRuntime((MLPolicyDefinition) policyDef);
+ }catch(Exception e){
+ LOG.error("ML Runtime creation failed: " + e.getMessage());
+ }
+ }
+
+ private MLRuntime newMLRuntime(MLPolicyDefinition mlPolicyDef) {
+ MLRuntime runtime = new MLRuntime();
+
+ try{
+ runtime.mlPolicyDef = mlPolicyDef;
+
+ LOG.info("policydef: " + ((runtime.mlPolicyDef == null)? "policy definition is null": "policy definition is not null"));
+ Properties alertContext = runtime.mlPolicyDef.getContext();
+ LOG.info("alert context received null? " + ((alertContext == null? "yes": "no")));
+ MLAnomalyCallback callbackImpl = new MLAnomalyCallbackImpl(this, config);
+ runtime.mlAnomalyCallbacks.add(callbackImpl);
+
+ MLAlgorithm[] mlAlgorithms = mlPolicyDef.getAlgorithms();
+ runtime.mlAlgorithmEvaluators = new MLAlgorithmEvaluator[mlAlgorithms.length];
+ LOG.info("mlAlgorithms size:: " + mlAlgorithms.length);
+ int i = 0;
+ for(MLAlgorithm algorithm:mlAlgorithms){
+ MLAlgorithmEvaluator mlAlgorithmEvaluator = MLReflectionUtils.newMLAlgorithmEvaluator(algorithm);
+ mlAlgorithmEvaluator.init(algorithm,config);
+ runtime.mlAlgorithmEvaluators[i] = mlAlgorithmEvaluator;
+ LOG.info("mlAlgorithmEvaluator: " + mlAlgorithmEvaluator.toString());
+ mlAlgorithmEvaluator.register(callbackImpl);
+ i++;
+ }
+ }catch(Exception ex){
+ LOG.error("Failed to create runtime for policy named: "+this.getPolicyName(),ex);
+ }
+ return runtime;
+ }
+
+ @Override
+ public void evaluate(ValuesArray data) throws Exception {
+ LOG.info("Evaluate called with input: " + data.size());
+ synchronized(mlRuntime){
+ for(MLAlgorithmEvaluator mlAlgorithm:mlRuntime.mlAlgorithmEvaluators){
+ mlAlgorithm.evaluate(data);
+ }
+ }
+ }
+
+ @Override
+ public void onPolicyUpdate(AlertDefinitionAPIEntity newAlertDef) {
+ LOG.info("onPolicyUpdate called");
+ AbstractPolicyDefinition policyDef = null;
+ try {
+ policyDef = JsonSerDeserUtils.deserialize(newAlertDef.getPolicyDef(),
+ AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(newAlertDef.getTags().get("policyType")));
+ } catch (Exception ex) {
+ LOG.error("initial policy def error, ", ex);
+ }
+ MLRuntime previous = mlRuntime;
+ mlRuntime = newMLRuntime((MLPolicyDefinition) policyDef);
+ synchronized (previous) {
+ previous.mlAnomalyCallbacks = null;
+ previous.mlAlgorithmEvaluators = null;
+ previous.mlPolicyDef = null;
+ }
+ previous = null;
+ }
+
+ @Override
+ public void onPolicyDelete() {
+ LOG.info("onPolicyDelete called");
+ MLRuntime previous = mlRuntime;
+ synchronized (previous) {
+ previous.mlAnomalyCallbacks = null;
+ previous.mlAlgorithmEvaluators = null;
+ previous.mlPolicyDef = null;
+ }
+ previous = null;
+ }
+
+ public String getPolicyName() {
+ return policyName;
+ }
+
+ public Map<String, String> getAdditionalContext() {
+ return this.context;
+ }
+
+ public List<String> getOutputStreamAttrNameList() {
+ return new ArrayList<String>();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java
new file mode 100644
index 0000000..4684d8d
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLAnomalyCallbackImpl.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.impl;
+
+import org.apache.eagle.alert.common.AlertConstants;
+import org.apache.eagle.alert.entity.AlertAPIEntity;
+import org.apache.eagle.alert.siddhi.EagleAlertContext;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.common.config.EagleConfigConstants;
+import org.apache.eagle.ml.MLAnomalyCallback;
+import org.apache.eagle.ml.MLPolicyEvaluator;
+import org.apache.eagle.ml.model.MLCallbackResult;
+import org.apache.eagle.common.metric.AlertContext;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.ManagementFactory;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MLAnomalyCallbackImpl implements MLAnomalyCallback {
+ private static Logger LOG = LoggerFactory.getLogger(MLAnomalyCallbackImpl.class);
+ private MLPolicyEvaluator mlAlertEvaluator;
+ private Config config;
+
+
+ public static final String source = ManagementFactory.getRuntimeMXBean().getName();
+
+ public MLAnomalyCallbackImpl(MLPolicyEvaluator mlAlertEvaluator, Config config){
+ this.mlAlertEvaluator = mlAlertEvaluator;
+ this.config = config;
+ }
+
+ /**
+ * TODO: generate alert
+ *
+ * @param aResult
+ * @param alertContext context
+ */
+ @Override
+ public void receive(MLCallbackResult aResult,EagleAlertContext alertContext) {
+ LOG.info("Receive called with : " + aResult.toString());
+ AlertAPIEntity alert = renderAlert(aResult,alertContext);
+ alertContext.alertExecutor.onAlerts(alertContext, Arrays.asList(alert));
+ }
+
+ private AlertAPIEntity renderAlert(MLCallbackResult aResult,EagleAlertContext alertContext){
+ String site = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.SITE);
+ String dataSource = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.DATA_SOURCE);
+
+ AlertAPIEntity entity = new AlertAPIEntity();
+ entity.setDescription(aResult.toString());
+
+ Map<String, String> tags = new HashMap<>();
+ tags.put(EagleConfigConstants.SITE, site);
+ tags.put(EagleConfigConstants.DATA_SOURCE, dataSource);
+ tags.put(AlertConstants.SOURCE_STREAMS, alertContext.evaluator.getAdditionalContext().get(AlertConstants.SOURCE_STREAMS));
+ tags.put(AlertConstants.POLICY_ID, alertContext.policyId);
+ tags.put(AlertConstants.ALERT_SOURCE, source);
+ tags.put(AlertConstants.ALERT_EXECUTOR_ID, alertContext.alertExecutor.getAlertExecutorId());
+ entity.setTags(tags);
+
+ entity.setTimestamp(aResult.getTimestamp());
+
+ AlertContext context = new AlertContext();
+
+ if(aResult.getContext() != null) context.addAll(aResult.getContext());
+
+ String alertMessage = "Anomaly activities detected by algorithm ["+aResult.getAlgorithmName()+"] with information: " + aResult.toString() ;
+ context.addProperty(AlertConstants.ALERT_EVENT, aResult.toString());
+ context.addProperty(AlertConstants.ALERT_MESSAGE, alertMessage);
+ context.addProperty(AlertConstants.ALERT_TIMESTAMP_PROPERTY, DateTimeUtil.millisecondsToHumanDateWithSeconds(System.currentTimeMillis()));
+
+ try {
+ site = config.getString("eagleProps.site");
+ dataSource = config.getString("eagleProps.dataSource");
+ context.addProperty(EagleConfigConstants.DATA_SOURCE, dataSource);
+ context.addProperty(EagleConfigConstants.SITE, site);
+ } catch (Exception ex) {
+ LOG.error("site, dataSource not set in config file, ", ex);
+ }
+
+ context.addProperty(EagleConfigConstants.DATA_SOURCE, dataSource);
+ context.addProperty(EagleConfigConstants.SITE, site);
+ context.addProperty(AlertConstants.POLICY_NAME, alertContext.policyId);
+
+ entity.setAlertContext(context);
+ return entity;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLModelDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLModelDAOImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLModelDAOImpl.java
new file mode 100644
index 0000000..ce6869c
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLModelDAOImpl.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.impl;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.alert.dao.AlertStreamSchemaDAOImpl;
+import org.apache.eagle.common.config.EagleConfigConstants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.ml.MLConstants;
+import org.apache.eagle.ml.MLModelDAO;
+import org.apache.eagle.ml.model.MLModelAPIEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.apache.commons.lang.time.DateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class MLModelDAOImpl implements MLModelDAO {
+ private final Logger LOG = LoggerFactory.getLogger(AlertStreamSchemaDAOImpl.class);
+
+ private String eagleServiceHost;
+ private int eagleServicePort;
+ private String eagleServiceUserName;
+ private String eagleServicePassword;
+
+ public MLModelDAOImpl(String eagleServiceHost, int eagleServicePort){
+ this.eagleServiceHost = eagleServiceHost;
+ this.eagleServicePort = eagleServicePort;
+ }
+
+ public MLModelDAOImpl(String eagleServiceHost, int eagleServicePort, String eagleServiceUserName, String eagleServicePassword){
+ this.eagleServiceHost = eagleServiceHost;
+ this.eagleServicePort = eagleServicePort;
+ this.eagleServiceUserName = eagleServiceUserName;
+ this.eagleServicePassword = eagleServicePassword;
+ }
+
+ public MLModelDAOImpl(Config config){
+ this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
+ this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
+ this.eagleServiceUserName =config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
+ this.eagleServicePassword = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD);
+ }
+
+ @Override
+ public List<MLModelAPIEntity> findMLModelByContext(String user, String algorithm) {
+ try {
+ IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, eagleServiceUserName, eagleServicePassword);
+ String query = MLConstants.ML_MODEL_SERVICE_NAME + "[@user=\"" + user + "\" AND @algorithm=\""
+ + algorithm + "\"]{*}";
+ GenericServiceAPIResponseEntity<MLModelAPIEntity> response = client.search().startTime(0)
+ .endTime(10 * DateUtils.MILLIS_PER_DAY)
+ .pageSize(Integer.MAX_VALUE)
+ .query(query)
+ .send();
+ if(!response.isSuccess()) {
+ LOG.error(String.format("Failed to get model for user: %s, algorithm: %s, due to: %s",user,algorithm,response.getException()));
+ }
+
+ client.close();
+ return response.getObj();
+ } catch (Exception ex) {
+ LOG.info("Got an exception when query machinelearning model service ", ex);
+ throw new IllegalStateException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
new file mode 100644
index 0000000..f61a902
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/impl/MLPolicyEvaluatorServiceProviderImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.impl;
+
+import org.apache.eagle.alert.common.AlertConstants;
+import org.apache.eagle.alert.policy.PolicyEvaluator;
+import org.apache.eagle.alert.policy.PolicyEvaluatorServiceProvider;
+import org.apache.eagle.ml.MLPolicyEvaluator;
+import org.apache.eagle.ml.model.MLPolicyDefinition;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+public class MLPolicyEvaluatorServiceProviderImpl implements PolicyEvaluatorServiceProvider {
+
+ private final static String ALERT_CONTEXT = "alertContext";
+
+ @Override
+ public String getPolicyType() {
+ return AlertConstants.policyType.MachineLearning.name();
+ }
+
+ @Override
+ public Class<? extends PolicyEvaluator> getPolicyEvaluator() {
+ return MLPolicyEvaluator.class;
+ }
+
+ @Override
+ public List<Module> getBindingModules() {
+ Module module1 = new SimpleModule(AlertConstants.POLICY_DEFINITION).registerSubtypes(new NamedType(MLPolicyDefinition.class, getPolicyType()));
+ Module module2 = new SimpleModule(ALERT_CONTEXT).registerSubtypes(new NamedType(Properties.class, getPolicyType()));
+ return Arrays.asList(module1, module2);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLAlgorithm.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLAlgorithm.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLAlgorithm.java
new file mode 100644
index 0000000..24b4bad
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLAlgorithm.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.model;
+
+import java.io.Serializable;
+
+public class MLAlgorithm implements Serializable {
+ private static final long serialVersionUID = -223531147520123L;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ private String name;
+ private String evaluator;
+ private String description;
+
+ public String getFeatures() {
+ return features;
+ }
+
+ public void setFeatures(String features) {
+ this.features = features;
+ }
+
+ private String features;
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getEvaluator() {
+ return evaluator;
+ }
+
+ public void setEvaluator(String evaluator) {
+ this.evaluator = evaluator;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLCallbackResult.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLCallbackResult.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLCallbackResult.java
new file mode 100644
index 0000000..b457973
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLCallbackResult.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class MLCallbackResult {
+ private boolean isAnomaly;
+ private List<String> feature;
+ private double confidence;
+ private long timestamp;
+ private List<String> datapoints;
+ private String id;
+ private String algorithmName;
+
+ public Map<String, String> getContext() {
+ return context;
+ }
+
+ public void setContext(Map<String, String> context) {
+ this.context = context;
+ }
+
+ public void setAlgorithmName(String algorithmName) {
+ this.algorithmName = algorithmName;
+ }
+
+ private Map<String,String> context;
+
+ public String getAlgorithmName() {
+ return algorithmName;
+ }
+ public void setAlgorithm(String algorithmName) {
+ this.algorithmName = algorithmName;
+ }
+ public MLCallbackResult(){
+ feature = new ArrayList<String>();
+ setDatapoints(new ArrayList<String>());
+ }
+ public boolean isAnomaly() {
+ return isAnomaly;
+ }
+ public void setAnomaly(boolean isAnomaly) {
+ this.isAnomaly = isAnomaly;
+ }
+ public List<String> getFeature() {
+ return feature;
+ }
+ public void setFeature(List<String> feature) {
+ this.feature = feature;
+ }
+
+ private boolean doesFeatureExist(String f){
+ boolean alreadyExist = false;
+ for(String s:feature){
+ if(s.equalsIgnoreCase(f))
+ alreadyExist = true;
+ }
+ return alreadyExist;
+ }
+ public void setFeature(String aFeature) {
+ if(doesFeatureExist(aFeature) == false)
+ feature.add(aFeature);
+ }
+ public double getConfidence() {
+ return confidence;
+ }
+ public void setConfidence(double confidence) {
+ this.confidence = confidence;
+ }
+ public String toString(){
+ StringBuffer resultStr = new StringBuffer();
+ resultStr.append("datapoint :<");
+ int i=0;
+ for(String d:datapoints){
+ if(i < datapoints.size()-1){
+ resultStr.append(d.trim());
+ resultStr.append(",");
+ i++;
+ }else
+ resultStr.append(d.trim());
+ }
+ resultStr.append("> for user: ").append(id).append(" is anomaly: ").append(this.isAnomaly).append(" at timestamp: ").append(this.timestamp);
+ if(this.isAnomaly){
+ //resultStr.append(" with confidence: " + this.confidence);
+ resultStr.append(" with algorithm: " + algorithmName);
+ resultStr.append(" with features: ");
+ int p=0;
+ resultStr.append("[");
+ for(String s:feature){
+ if(p < feature.size()-1){
+ resultStr.append(s.trim());
+ resultStr.append(",");
+ p++;
+ }else
+ resultStr.append(s.trim());
+ }
+ resultStr.append("]");
+ }
+ return resultStr.toString();
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+ public List<String> getDatapoints() {
+ return datapoints;
+ }
+ public void setDatapoints(List<String> datapoints) {
+ this.datapoints = datapoints;
+ }
+ public String getId() {
+ return id;
+ }
+ public void setId(String id) {
+ this.id = id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLEntityRepository.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLEntityRepository.java
new file mode 100644
index 0000000..883847a
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLEntityRepository.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.model;
+
+import org.apache.eagle.log.entity.repo.EntityRepository;
+
+public class MLEntityRepository extends EntityRepository {
+ public MLEntityRepository() {
+ entitySet.add(MLModelAPIEntity.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLModelAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLModelAPIEntity.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLModelAPIEntity.java
new file mode 100644
index 0000000..6f91ca6
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLModelAPIEntity.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.model;
+
+import org.apache.eagle.ml.MLConstants;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.Column;
+import org.apache.eagle.log.entity.meta.ColumnFamily;
+import org.apache.eagle.log.entity.meta.Prefix;
+import org.apache.eagle.log.entity.meta.Service;
+import org.apache.eagle.log.entity.meta.Table;
+import org.apache.eagle.log.entity.meta.Tags;
+import org.apache.eagle.log.entity.meta.TimeSeries;
+
+/**
+ * DDL for creating the table
+ * create 'mlmodel', {NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', COMPRESSION => 'SNAPPY'}
+ */
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@Table("mlmodel")
+@ColumnFamily("f")
+@Prefix("mlmodel")
+@Service(MLConstants.ML_MODEL_SERVICE_NAME)
+@JsonIgnoreProperties(ignoreUnknown = true)
+@TimeSeries(false)
+@Tags({"site","user", "algorithm"})
+public class MLModelAPIEntity extends TaggedLogAPIEntity{
+ @Column("b")
+ private String content;
+ @Column("c")
+ private long version;
+
+ public String getContent() {
+ return content;
+ }
+
+ public void setContent(String content) {
+ this.content = content;
+ valueChanged("content");
+ }
+
+ public long getVersion() {
+ return version;
+ }
+
+ public void setVersion(long version) {
+ this.version = version;
+ valueChanged("version");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLPolicyDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLPolicyDefinition.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLPolicyDefinition.java
new file mode 100644
index 0000000..71fabbf
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/model/MLPolicyDefinition.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.model;
+
+import org.apache.eagle.alert.config.AbstractPolicyDefinition;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+import java.util.Properties;
+
+/**
+ {
+ "type":"MachineLearning",
+ "context":{
+ "site":"dev",
+ "dataSource":"userprofile"
+ "component":"dev-component",
+ "description":"ML based user profile anomaly detection",
+ "severity":"WARNING",
+ "notificationByEmail":"true"
+ },
+ "algorithms":[
+ {
+ "name":"eagle.security.userprofile.util.EigenBasedAnomalyDetection",
+ "description":"EigenBasedAnomalyDetection",
+ "features":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete"
+ },
+ {
+ "name":"eagle.security.userprofile.util.KDEBasedAnomalyDetection",
+ "description":"DensityBasedAnomalyDetection",
+ "features":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete"
+ }
+ ]
+}
+ version field is used for model update, so eagle framework can understand that something changes.
+
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible=true)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MLPolicyDefinition extends AbstractPolicyDefinition{
+ private String version;
+ private Properties context;
+ private MLAlgorithm[] algorithms;
+
+ public MLAlgorithm[] getAlgorithms() {
+ return algorithms;
+ }
+
+ public void setAlgorithms(MLAlgorithm[] algorithms) {
+ this.algorithms = algorithms;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public Properties getContext() {
+ return context;
+ }
+
+ public void setContext(Properties context) {
+ this.context = context;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/utils/MLReflectionUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/utils/MLReflectionUtils.java b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/utils/MLReflectionUtils.java
new file mode 100644
index 0000000..fe70165
--- /dev/null
+++ b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/java/org/apache/eagle/ml/utils/MLReflectionUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.ml.utils;
+
+import org.apache.eagle.ml.MLAlgorithmEvaluator;
+import org.apache.eagle.ml.model.MLAlgorithm;
+
+/**
+ * @since 8/14/15
+ */
+public class MLReflectionUtils {
+ /**
+ * Create a new MLAlgorithmEvaluator based on MLAlgorithm
+ *
+ * @param algorithm MLAlgorithm
+ * @return MLAlgorithmEvaluator object instance
+ * @throws ClassNotFoundException
+ * @throws IllegalAccessException
+ * @throws InstantiationException
+ */
+ public static MLAlgorithmEvaluator newMLAlgorithmEvaluator(MLAlgorithm algorithm) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+ return (MLAlgorithmEvaluator) Class.forName(algorithm.getEvaluator()).newInstance();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/afe86834/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/resources/META-INF/services/eagle.alert.policy.PolicyEvaluatorServiceProvider
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/resources/META-INF/services/eagle.alert.policy.PolicyEvaluatorServiceProvider b/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/resources/META-INF/services/eagle.alert.policy.PolicyEvaluatorServiceProvider
deleted file mode 100644
index 6d6a3d7..0000000
--- a/eagle-core/eagle-machinelearning/eagle-machinelearning-base/src/main/resources/META-INF/services/eagle.alert.policy.PolicyEvaluatorServiceProvider
+++ /dev/null
@@ -1 +0,0 @@
-eagle.ml.impl.MLPolicyEvaluatorServiceProviderImpl
\ No newline at end of file