You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/03/02 20:51:56 UTC
[10/10] incubator-metron git commit: METRON-503: Metron REST API this
closes apache/incubator-metron#316
METRON-503: Metron REST API this closes apache/incubator-metron#316
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e6628499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e6628499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e6628499
Branch: refs/heads/master
Commit: e662849932a74187708b0037042c6b06d4750116
Parents: a61dbcf
Author: merrimanr <me...@gmail.com>
Authored: Thu Mar 2 11:04:06 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Mar 2 15:24:11 2017 -0500
----------------------------------------------------------------------
dependencies_with_url.csv | 3 +
metron-interface/metron-rest-client/pom.xml | 2 +-
.../metron/rest/model/GrokValidation.java | 13 +-
.../metron/rest/model/ParseMessageRequest.java | 9 +
.../metron/rest/model/SensorParserContext.java | 51 +++
.../metron/rest/model/TopologyResponse.java | 18 +
.../metron/rest/model/TopologyStatus.java | 28 +-
.../metron/rest/model/TopologySummary.java | 17 +
.../rest/model/TransformationValidation.java | 51 ---
metron-interface/metron-rest/.gitignore | 1 +
metron-interface/metron-rest/README.md | 222 ++++++++----
metron-interface/metron-rest/pom.xml | 39 +--
.../metron/rest/controller/HdfsController.java | 89 +++++
.../rest/controller/StellarController.java | 80 +++++
.../controller/TransformationController.java | 80 -----
.../apache/metron/rest/service/GrokService.java | 3 +
.../apache/metron/rest/service/HdfsService.java | 13 +-
.../rest/service/SensorParserConfigService.java | 3 +
.../metron/rest/service/StellarService.java | 39 +++
.../rest/service/TransformationService.java | 39 ---
.../rest/service/impl/GrokServiceImpl.java | 59 +++-
.../rest/service/impl/HdfsServiceImpl.java | 55 ++-
.../impl/SensorEnrichmentConfigServiceImpl.java | 8 +-
.../impl/SensorIndexingConfigServiceImpl.java | 8 +-
.../impl/SensorParserConfigServiceImpl.java | 141 +-------
.../rest/service/impl/StellarServiceImpl.java | 92 +++++
.../service/impl/StormAdminServiceImpl.java | 13 +-
.../rest/service/impl/StormCLIWrapper.java | 6 +-
.../service/impl/StormStatusServiceImpl.java | 8 +-
.../service/impl/TransformationServiceImpl.java | 92 -----
.../src/main/resources/application-docker.yml | 6 +-
.../src/main/resources/application-vagrant.yml | 51 +++
.../src/main/resources/application.yml | 2 +-
.../metron-rest/src/main/scripts/start.sh | 33 --
.../src/main/scripts/start_metron_rest.sh | 25 ++
.../GlobalConfigControllerIntegrationTest.java | 8 +
.../GrokControllerIntegrationTest.java | 24 +-
.../HdfsControllerIntegrationTest.java | 101 ++++++
...richmentConfigControllerIntegrationTest.java | 23 +-
...IndexingConfigControllerIntegrationTest.java | 2 +
...orParserConfigControllerIntegrationTest.java | 36 +-
.../StellarControllerIntegrationTest.java | 122 +++++++
.../StormControllerIntegrationTest.java | 3 +
...TransformationControllerIntegrationTest.java | 122 -------
.../rest/service/SensorParserConfigTest.java | 116 -------
.../service/impl/DockerStormCLIWrapperTest.java | 14 +-
.../rest/service/impl/GrokServiceImplTest.java | 136 ++++++--
.../impl/HdfsServiceImplExceptionTest.java | 101 ++++++
.../rest/service/impl/HdfsServiceImplTest.java | 143 ++++----
.../rest/service/impl/KafkaServiceImplTest.java | 4 +-
.../SensorEnrichmentConfigServiceImplTest.java | 256 ++++++++++++++
.../SensorIndexingConfigServiceImplTest.java | 234 +++++++++++++
.../impl/SensorParserConfigServiceImplTest.java | 344 +++++++++++++++++++
.../service/impl/StellarServiceImplTest.java | 92 +++++
.../service/impl/StormAdminServiceImplTest.java | 156 +++++++++
.../rest/service/impl/StormCLIWrapperTest.java | 216 ++++++++++++
.../impl/StormStatusServiceImplTest.java | 221 ++++++++++++
.../metron-rest/src/test/resources/README.vm | 121 +++++--
metron-interface/pom.xml | 2 +-
59 files changed, 3013 insertions(+), 983 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 35acced..1d428c3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -283,3 +283,6 @@ org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,AS
org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org
+com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html
+de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html
+org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
index 66ccf52..0a49986 100644
--- a/metron-interface/metron-rest-client/pom.xml
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.3.0</version>
+ <version>0.3.1</version>
</parent>
<artifactId>metron-rest-client</artifactId>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
index ccd2c5c..2795320 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java
@@ -22,10 +22,19 @@ import java.util.Map;
public class GrokValidation {
+ private String patternLabel;
private String statement;
private String sampleData;
private Map<String, Object> results;
+ public String getPatternLabel() {
+ return patternLabel;
+ }
+
+ public void setPatternLabel(String patternLabel) {
+ this.patternLabel = patternLabel;
+ }
+
public String getStatement() {
return statement;
}
@@ -60,6 +69,7 @@ public class GrokValidation {
GrokValidation that = (GrokValidation) o;
+ if (patternLabel != null ? !patternLabel.equals(that.patternLabel) : that.patternLabel != null) return false;
if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false;
if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false;
return results != null ? results.equals(that.results) : that.results == null;
@@ -67,7 +77,8 @@ public class GrokValidation {
@Override
public int hashCode() {
- int result = statement != null ? statement.hashCode() : 0;
+ int result = patternLabel != null ? patternLabel.hashCode() : 0;
+ result = 31 * result + (statement != null ? statement.hashCode() : 0);
result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0);
result = 31 * result + (results != null ? results.hashCode() : 0);
return result;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
index 8dfe17b..50eb88d 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java
@@ -22,6 +22,7 @@ import org.apache.metron.common.configuration.SensorParserConfig;
public class ParseMessageRequest {
private SensorParserConfig sensorParserConfig;
+ private String grokStatement;
private String sampleData;
public SensorParserConfig getSensorParserConfig() {
@@ -32,6 +33,14 @@ public class ParseMessageRequest {
this.sensorParserConfig = sensorParserConfig;
}
+ public String getGrokStatement() {
+ return grokStatement;
+ }
+
+ public void setGrokStatement(String grokStatement) {
+ this.grokStatement = grokStatement;
+ }
+
public String getSampleData() {
return sampleData;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
new file mode 100644
index 0000000..cb10cfe
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SensorParserContext {
+
+ private Map<String, Object> sampleData;
+ private SensorParserConfig sensorParserConfig;
+
+ public Map<String, Object> getSampleData() {
+ if (sampleData == null) {
+ return new HashMap<>();
+ }
+ return sampleData;
+ }
+
+ public void setSampleData(Map<String, Object> sampleData) {
+ this.sampleData = sampleData;
+ }
+
+ public SensorParserConfig getSensorParserConfig() {
+ if (sensorParserConfig == null) {
+ return new SensorParserConfig();
+ }
+ return sensorParserConfig;
+ }
+
+ public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
+ this.sensorParserConfig = sensorParserConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
index 6894e89..0ad2e9f 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java
@@ -39,4 +39,22 @@ public class TopologyResponse {
this.status = TopologyResponseCode.ERROR;
this.message = message;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologyResponse that = (TopologyResponse) o;
+
+ if (status != null ? !status.equals(that.status) : that.status != null) return false;
+ return message != null ? message.equals(that.message) : that.message == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = status != null ? status.hashCode() : 0;
+ result = 31 * result + (message != null ? message.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
index 0e2d28a..6fc1c01 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java
@@ -26,8 +26,8 @@ public class TopologyStatus {
private String name;
private TopologyStatusCode status;
private Map<String, Object>[] topologyStats;
- private double latency = 0;
- private double throughput = 0;
+ private Double latency = 0.0;
+ private Double throughput = 0.0;
public String getId() {
return id;
@@ -73,4 +73,28 @@ public class TopologyStatus {
}
}
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologyStatus that = (TopologyStatus) o;
+
+ if (id != null ? !id.equals(that.id) : that.id != null) return false;
+ if (name != null ? !name.equals(that.name) : that.name != null) return false;
+ if (status != null ? !status.equals(that.status) : that.status != null) return false;
+ if (!latency.equals(that.latency)) return false;
+ return throughput.equals(that.throughput);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id != null ? id.hashCode() : 0;
+ result = 31 * result + (name != null ? name.hashCode() : 0);
+ result = 31 * result + (status != null ? status.hashCode() : 0);
+ result = 31 * result + (latency != null ? latency.hashCode() : 0);
+ result = 31 * result + (throughput != null ? throughput.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
index b9d39a4..8621daf 100644
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.rest.model;
+import java.util.Arrays;
+
public class TopologySummary {
private TopologyStatus[] topologies;
@@ -31,4 +33,19 @@ public class TopologySummary {
public void setTopologies(TopologyStatus[] topologies) {
this.topologies = topologies;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TopologySummary that = (TopologySummary) o;
+
+ return topologies != null ? Arrays.equals(topologies, that.topologies) : that.topologies != null;
+ }
+
+ @Override
+ public int hashCode() {
+ return topologies != null ? Arrays.hashCode(topologies) : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
deleted file mode 100644
index c2e39e4..0000000
--- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TransformationValidation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.model;
-
-import org.apache.metron.common.configuration.SensorParserConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class TransformationValidation {
-
- private Map<String, Object> sampleData;
- private SensorParserConfig sensorParserConfig;
-
- public Map<String, Object> getSampleData() {
- if (sampleData == null) {
- return new HashMap<>();
- }
- return sampleData;
- }
-
- public void setSampleData(Map<String, Object> sampleData) {
- this.sampleData = sampleData;
- }
-
- public SensorParserConfig getSensorParserConfig() {
- if (sensorParserConfig == null) {
- return new SensorParserConfig();
- }
- return sensorParserConfig;
- }
-
- public void setSensorParserConfig(SensorParserConfig sensorParserConfig) {
- this.sensorParserConfig = sensorParserConfig;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/.gitignore
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/.gitignore b/metron-interface/metron-rest/.gitignore
new file mode 100644
index 0000000..26b3af1
--- /dev/null
+++ b/metron-interface/metron-rest/.gitignore
@@ -0,0 +1 @@
+metrondb.*.db
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 63a2d34..cf41bf3 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -1,66 +1,83 @@
-# Metron REST and Configuration UI
+# Metron REST
-This UI exposes and aids in sensor configuration.
+This module provides a RESTful API for interacting with Metron.
## Prerequisites
* A running Metron cluster
-* A running instance of MySQL
* Java 8 installed
* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
## Installation
-1. Package the Application with Maven:
- ```
- mvn clean package
- ```
+1. Package the application with Maven:
+```
+mvn clean package
+```
1. Untar the archive in the target directory. The directory structure will look like:
- ```
- bin
- start.sh
- lib
- metron-rest-version.jar
- ```
+```
+bin
+ start_metron_rest.sh
+lib
+ metron-rest-$METRON_VERSION.jar
+```
+
+1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
-1. Install Hibernate by downloading version 5.0.11.Final from (http://hibernate.org/orm/downloads/). Unpack the archive and set the HIBERNATE_HOME environment variable to the absolute path of the top level directory.
- ```
- export HIBERNATE_HOME=/path/to/hibernate-release-5.0.11.Final
- ```
+1. Start the application with this command:
+```
+./bin/start_metron_rest.sh /path/to/application.yml
+```
-1. Install the MySQL client by downloading version 5.1.40 from (https://dev.mysql.com/downloads/connector/j/). Unpack the archive and set the MYSQL_CLIENT_HOME environment variable to the absolute path of the top level directory.
- ```
- export MYSQL_CLIENT_HOME=/path/to/mysql-connector-java-5.1.40
- ```
+## Usage
-1. Create a MySQL user for the Config UI (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
-1. Create a Config UI database in MySQL with this command:
- ```
- CREATE DATABASE IF NOT EXISTS metronrest
- ```
+## Security
-1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.username` and `spring.datasource.password` properties using the MySQL credentials from step 4.
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users:
+```
+./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+```
-1. Start the UI with this command:
- ```
- ./bin/start.sh /path/to/application.yml
- ```
+For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows:
-## Usage
+1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. Users can be added with this SQL statement:
+1. Connect to MySQL and create a Metron REST database:
+```
+CREATE DATABASE IF NOT EXISTS metronrest
+```
+
+1. Add users:
```
use metronrest;
insert into users (username, password, enabled) values ('your_username','your_password',1);
insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
```
-Users can be added to additional groups with this SQL statement:
+
+1. Replace the H2 connection information in the application.yml file with MySQL connection information:
```
-use metronrest;
-insert into authorities (username, authority) values ('your_username', 'your_group');
+spring:
+ datasource:
+ driverClassName: com.mysql.jdbc.Driver
+ url: jdbc:mysql://mysql_host:3306/metronrest
+ username: metron_rest_user
+ password: metron_rest_password
+ platform: mysql
```
+1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
+```
+<dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.client.version}</version>
+</dependency>
+```
+
+1. Follow the steps in the [Installation](#installation) section
+
## API
Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI.
@@ -72,6 +89,10 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `POST /api/v1/global/config`](#post-apiv1globalconfig)|
| [ `GET /api/v1/grok/list`](#get-apiv1groklist)|
| [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)|
+| [ `POST /api/v1/hdfs`](#post-apiv1hdfs)|
+| [ `GET /api/v1/hdfs`](#get-apiv1hdfs)|
+| [ `DELETE /api/v1/hdfs`](#delete-apiv1hdfs)|
+| [ `GET /api/v1/hdfs/list`](#get-apiv1hdfslist)|
| [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)|
| [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)|
| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})|
@@ -93,6 +114,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)|
| [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})|
| [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})|
+| [ `POST /api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)|
+| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)|
+| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)|
+| [ `GET /api/v1/stellar/list/simple/functions`](#get-apiv1stellarlistsimplefunctions)|
+| [ `POST /api/v1/stellar/validate/rules`](#post-apiv1stellarvalidaterules)|
| [ `GET /api/v1/storm`](#get-apiv1storm)|
| [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)|
| [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)|
@@ -110,11 +136,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})|
| [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})|
| [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})|
-| [ `GET /api/v1/transformation/list`](#get-apiv1transformationlist)|
-| [ `GET /api/v1/transformation/list/functions`](#get-apiv1transformationlistfunctions)|
-| [ `GET /api/v1/transformation/list/simple/functions`](#get-apiv1transformationlistsimplefunctions)|
-| [ `POST /api/v1/transformation/validate`](#post-apiv1transformationvalidate)|
-| [ `POST /api/v1/transformation/validate/rules`](#post-apiv1transformationvalidaterules)|
| [ `GET /api/v1/user`](#get-apiv1user)|
### `GET /api/v1/global/config`
@@ -145,10 +166,41 @@ Request and Response objects are JSON formatted. The JSON schemas are available
### `POST /api/v1/grok/validate`
* Description: Applies a Grok statement to a sample message
* Input:
- * grokValidation - Object containing Grok statment and sample message
+ * grokValidation - Object containing Grok statement and sample message
* Returns:
* 200 - JSON results
+### `POST /api/v1/hdfs`
+ * Description: Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.
+ * Input:
+ * path - Path to HDFS file
+ * contents - File contents
+ * Returns:
+ * 200 - Contents were written
+
+### `GET /api/v1/hdfs`
+ * Description: Reads a file from HDFS and returns the contents
+ * Input:
+ * path - Path to HDFS file
+ * Returns:
+ * 200 - Returns file contents
+
+### `DELETE /api/v1/hdfs`
+ * Description: Deletes a file from HDFS
+ * Input:
+ * path - Path to HDFS file
+ * recursive - Delete files recursively
+ * Returns:
+ * 200 - File was deleted
+ * 404 - File was not found in HDFS
+
+### `GET /api/v1/hdfs/list`
+ * Description: Reads a file from HDFS and returns the contents
+ * Input:
+ * path - Path to HDFS directory
+ * Returns:
+ * 200 - Returns file contents
+
### `GET /api/v1/kafka/topic`
* Description: Retrieves all Kafka topics
* Returns:
@@ -296,6 +348,35 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns SensorParserConfig
* 404 - SensorParserConfig is missing
+### `POST /api/v1/stellar/apply/transformations`
+ * Description: Executes transformations against a sample message
+ * Input:
+ * transformationValidation - Object containing SensorParserConfig and sample message
+ * Returns:
+ * 200 - Returns transformation results
+
+### `GET /api/v1/stellar/list`
+ * Description: Retrieves field transformations
+ * Returns:
+ * 200 - Returns a list field transformations
+
+### `GET /api/v1/stellar/list/functions`
+ * Description: Lists the Stellar functions that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of Stellar functions
+
+### `GET /api/v1/stellar/list/simple/functions`
+ * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
+ * Returns:
+ * 200 - Returns a list of simple Stellar functions
+
+### `POST /api/v1/stellar/validate/rules`
+ * Description: Tests Stellar statements to ensure they are well-formed
+ * Input:
+ * statements - List of statements to validate
+ * Returns:
+ * 200 - Returns validation results
+
### `GET /api/v1/storm`
* Description: Retrieves the status of all Storm topologies
* Returns:
@@ -399,40 +480,43 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns topology status information
* 404 - Topology is missing
-### `GET /api/v1/transformation/list`
- * Description: Retrieves field transformations
+### `GET /api/v1/user`
+ * Description: Retrieves the current user
* Returns:
- * 200 - Returns a list field transformations
+ * 200 - Current user
-### `GET /api/v1/transformation/list/functions`
- * Description: Lists the Stellar functions that can be found on the classpath
- * Returns:
- * 200 - Returns a list of Stellar functions
+## Testing
-### `GET /api/v1/transformation/list/simple/functions`
- * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath
- * Returns:
- * 200 - Returns a list of simple Stellar functions
+Profiles are includes for both the metron-docker and Quick Dev environments.
-### `POST /api/v1/transformation/validate`
- * Description: Executes transformations against a sample message
- * Input:
- * transformationValidation - Object containing SensorParserConfig and sample message
- * Returns:
- * 200 - Returns transformation results
+### metron-docker
-### `POST /api/v1/transformation/validate/rules`
- * Description: Tests Stellar statements to ensure they are well-formed
- * Input:
- * statements - List of statements to validate
- * Returns:
- * 200 - Returns validation results
+Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=docker,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
-### `GET /api/v1/user`
- * Description: Retrieves the current user
- * Returns:
- * 200 - Current user
+### Quick Dev
+Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+```
+mvn clean package
+mvn spring-boot:run -Drun.profiles=vagrant,dev
+```
+The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
+
+To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
+```
+mvn clean package
+scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/
+```
+Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari:
+```
+java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+```
+The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.
## License
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index c2beb2d..849455e 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.3.0</version>
+ <version>0.3.1</version>
</parent>
<artifactId>metron-rest</artifactId>
<properties>
@@ -61,15 +61,15 @@
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.client.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -105,6 +105,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -122,8 +126,16 @@
</exclusion>
<exclusion>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-writer</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -147,18 +159,6 @@
<version>0.1.0</version>
</dependency>
<dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>0.9.10</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>annotations</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
@@ -166,7 +166,6 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
new file mode 100644
index 0000000..bf7ae84
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.HdfsService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+@RestController
+@RequestMapping("/api/v1/hdfs")
+public class HdfsController {
+
+ @Autowired
+ private HdfsService hdfsService;
+
+ @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+ @ApiResponse(message = "Returns file contents", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<List<String>> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException {
+ return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Reads a file from HDFS and returns the contents")
+ @ApiResponse(message = "Returns file contents", code = 200)
+ @RequestMapping(method = RequestMethod.GET)
+ ResponseEntity<String> read(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path) throws RestException {
+ String contents = hdfsService.read(new Path(path));
+ if (contents != null) {
+ return new ResponseEntity<>(hdfsService.read(new Path(path)), HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+
+ }
+
+ @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.")
+ @ApiResponse(message = "Contents were written", code = 200)
+ @RequestMapping(method = RequestMethod.POST)
+ ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path,
+ @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException {
+ hdfsService.write(new Path(path), contents.getBytes(UTF_8));
+ return new ResponseEntity<>(HttpStatus.OK);
+
+ }
+
+ @ApiOperation(value = "Deletes a file from HDFS")
+ @ApiResponses(value = { @ApiResponse(message = "File was deleted", code = 200),
+ @ApiResponse(message = "File was not found in HDFS", code = 404) })
+ @RequestMapping(method = RequestMethod.DELETE)
+ ResponseEntity<Boolean> delete(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path,
+ @ApiParam(name = "recursive", value = "Delete files recursively") @RequestParam(required = false, defaultValue = "false") boolean recursive) throws RestException {
+ if (hdfsService.delete(new Path(path), recursive)) {
+ return new ResponseEntity<>(HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
new file mode 100644
index 0000000..13327f9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/v1/stellar")
+public class StellarController {
+
+ @Autowired
+ private StellarService stellarService;
+
+ @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
+ @ApiResponse(message = "Returns validation results", code = 200)
+ @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Boolean>> validateRules(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
+ return new ResponseEntity<>(stellarService.validateRules(statements), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Executes transformations against a sample message")
+ @ApiResponse(message = "Returns transformation results", code = 200)
+ @RequestMapping(value = "/apply/transformations", method = RequestMethod.POST)
+ ResponseEntity<Map<String, Object>> applyTransformations(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody SensorParserContext sensorParserContext) throws RestException {
+ return new ResponseEntity<>(stellarService.applyTransformations(sensorParserContext), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Retrieves field transformations")
+ @ApiResponse(message = "Returns a list field transformations", code = 200)
+ @RequestMapping(value = "/list", method = RequestMethod.GET)
+ ResponseEntity<FieldTransformations[]> list() throws RestException {
+ return new ResponseEntity<>(stellarService.getTransformations(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
+ @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
+ return new ResponseEntity<>(stellarService.getStellarFunctions(), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
+ @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
+ @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
+ ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
+ return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
deleted file mode 100644
index bc8b201..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/TransformationController.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.controller;
-
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import io.swagger.annotations.ApiResponse;
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.RestException;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-import org.apache.metron.rest.service.TransformationService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.List;
-import java.util.Map;
-
-@RestController
-@RequestMapping("/api/v1/transformation")
-public class TransformationController {
-
- @Autowired
- private TransformationService transformationService;
-
- @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed")
- @ApiResponse(message = "Returns validation results", code = 200)
- @RequestMapping(value = "/validate/rules", method = RequestMethod.POST)
- ResponseEntity<Map<String, Boolean>> validateRule(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List<String> statements) throws RestException {
- return new ResponseEntity<>(transformationService.validateRules(statements), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Executes transformations against a sample message")
- @ApiResponse(message = "Returns transformation results", code = 200)
- @RequestMapping(value = "/validate", method = RequestMethod.POST)
- ResponseEntity<Map<String, Object>> validateTransformation(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody TransformationValidation transformationValidation) throws RestException {
- return new ResponseEntity<>(transformationService.validateTransformation(transformationValidation), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Retrieves field transformations")
- @ApiResponse(message = "Returns a list field transformations", code = 200)
- @RequestMapping(value = "/list", method = RequestMethod.GET)
- ResponseEntity<FieldTransformations[]> list() throws RestException {
- return new ResponseEntity<>(transformationService.getTransformations(), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath")
- @ApiResponse(message = "Returns a list of Stellar functions", code = 200)
- @RequestMapping(value = "/list/functions", method = RequestMethod.GET)
- ResponseEntity<List<StellarFunctionDescription>> listFunctions() throws RestException {
- return new ResponseEntity<>(transformationService.getStellarFunctions(), HttpStatus.OK);
- }
-
- @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath")
- @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200)
- @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET)
- ResponseEntity<List<StellarFunctionDescription>> listSimpleFunctions() throws RestException {
- return new ResponseEntity<>(transformationService.getSimpleStellarFunctions(), HttpStatus.OK);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index 95ce1ba..268d396 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.service;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
+import java.io.File;
import java.util.Map;
public interface GrokService {
@@ -28,4 +29,6 @@ public interface GrokService {
GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
+ File saveTemporary(String statement, String name) throws RestException;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index 97888ff..d5932c7 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -17,19 +17,18 @@
*/
package org.apache.metron.rest.service;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.springframework.stereotype.Service;
+import org.apache.metron.rest.RestException;
-import java.io.IOException;
+import java.util.List;
public interface HdfsService {
- byte[] read(Path path) throws IOException;
+ String read(Path path) throws RestException;
- void write(Path path, byte[] contents) throws IOException;
+ void write(Path path, byte[] contents) throws RestException;
- FileStatus[] list(Path path) throws IOException;
+ List<String> list(Path path) throws RestException;
- boolean delete(Path path, boolean recursive) throws IOException;
+ boolean delete(Path path, boolean recursive) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
index efda639..9b863b8 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java
@@ -22,6 +22,7 @@ import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.json.simple.JSONObject;
+import java.util.List;
import java.util.Map;
public interface SensorParserConfigService {
@@ -32,6 +33,8 @@ public interface SensorParserConfigService {
Iterable<SensorParserConfig> getAll() throws RestException;
+ List<String> getAllTypes() throws RestException;
+
boolean delete(String name) throws RestException;
Map<String, String> getAvailableParsers();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
new file mode 100644
index 0000000..14cd31f
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+
+import java.util.List;
+import java.util.Map;
+
+public interface StellarService {
+
+ Map<String, Boolean> validateRules(List<String> rules);
+
+ Map<String, Object> applyTransformations(SensorParserContext sensorParserContext);
+
+ FieldTransformations[] getTransformations();
+
+ List<StellarFunctionDescription> getStellarFunctions();
+
+ List<StellarFunctionDescription> getSimpleStellarFunctions();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
deleted file mode 100644
index d1400c6..0000000
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/TransformationService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.rest.service;
-
-import org.apache.metron.common.field.transformation.FieldTransformations;
-import org.apache.metron.rest.model.StellarFunctionDescription;
-import org.apache.metron.rest.model.TransformationValidation;
-
-import java.util.List;
-import java.util.Map;
-
-public interface TransformationService {
-
- Map<String, Boolean> validateRules(List<String> rules);
-
- Map<String, Object> validateTransformation(TransformationValidation transformationValidation);
-
- FieldTransformations[] getTransformations();
-
- List<StellarFunctionDescription> getStellarFunctions();
-
- List<StellarFunctionDescription> getSimpleStellarFunctions();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 323ca78..3f2de2f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -20,22 +20,35 @@ package org.apache.metron.rest.service.impl;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.directory.api.util.Strings;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Map;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+
@Service
public class GrokServiceImpl implements GrokService {
+
+ private Environment environment;
+
private Grok commonGrok;
@Autowired
- public GrokServiceImpl(Grok commonGrok) {
+ public GrokServiceImpl(Environment environment, Grok commonGrok) {
+ this.environment = environment;
this.commonGrok = commonGrok;
}
@@ -48,20 +61,21 @@ public class GrokServiceImpl implements GrokService {
public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException {
Map<String, Object> results;
try {
- String statement = Strings.isEmpty(grokValidation.getStatement()) ? "" : grokValidation.getStatement();
-
+ if (grokValidation.getPatternLabel() == null) {
+ throw new RestException("Pattern label is required");
+ }
+ if (Strings.isEmpty(grokValidation.getStatement())) {
+ throw new RestException("Grok statement is required");
+ }
Grok grok = new Grok();
grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common")));
- grok.addPatternFromReader(new StringReader(statement));
- String patternLabel = statement.substring(0, statement.indexOf(" "));
- String grokPattern = "%{" + patternLabel + "}";
+ grok.addPatternFromReader(new StringReader(grokValidation.getStatement()));
+ String grokPattern = "%{" + grokValidation.getPatternLabel() + "}";
grok.compile(grokPattern);
Match gm = grok.match(grokValidation.getSampleData());
gm.captures();
results = gm.toMap();
- results.remove(patternLabel);
- } catch (StringIndexOutOfBoundsException e) {
- throw new RestException("A pattern label must be included (eg. PATTERN_LABEL %{PATTERN:field} ...)", e.getCause());
+ results.remove(grokValidation.getPatternLabel());
} catch (Exception e) {
throw new RestException(e);
}
@@ -69,4 +83,31 @@ public class GrokServiceImpl implements GrokService {
return grokValidation;
}
+ @Override
+ public File saveTemporary(String statement, String name) throws RestException {
+ if (statement != null) {
+ try {
+ File grokDirectory = new File(getTemporaryGrokRootPath());
+ if (!grokDirectory.exists()) {
+ grokDirectory.mkdirs();
+ }
+ File path = new File(grokDirectory, name);
+ FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
+ fileWriter.write(statement);
+ fileWriter.close();
+ return path;
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ } else {
+ throw new RestException("A grokStatement must be provided");
+ }
+ }
+
+ private String getTemporaryGrokRootPath() {
+ String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
+ Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+ return new Path(grokTempPath, authentication.getName()).toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index c14ec0c..789c421 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -19,44 +19,73 @@ package org.apache.metron.rest.service.impl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.HdfsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
@Service
public class HdfsServiceImpl implements HdfsService {
- @Autowired
private Configuration configuration;
+ @Autowired
+ public HdfsServiceImpl(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
@Override
- public byte[] read(Path path) throws IOException {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
- return byteArrayOutputStream.toByteArray();
+ public List<String> list(Path path) throws RestException {
+ try {
+ return Arrays.asList(FileSystem.get(configuration).listStatus(path)).stream().map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
@Override
- public void write(Path path, byte[] contents) throws IOException {
- FSDataOutputStream fsDataOutputStream = FileSystem.get(configuration).create(path, true);
- fsDataOutputStream.write(contents);
- fsDataOutputStream.close();
+ public String read(Path path) throws RestException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ try {
+ IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration);
+ } catch (FileNotFoundException e) {
+ return null;
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ return new String(byteArrayOutputStream.toByteArray(), UTF_8);
}
@Override
- public FileStatus[] list(Path path) throws IOException {
- return FileSystem.get(configuration).listStatus(path);
+ public void write(Path path, byte[] contents) throws RestException {
+ FSDataOutputStream fsDataOutputStream;
+ try {
+ fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+ fsDataOutputStream.write(contents);
+ fsDataOutputStream.close();
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
@Override
- public boolean delete(Path path, boolean recursive) throws IOException {
+ public boolean delete(Path path, boolean recursive) throws RestException {
+ try {
return FileSystem.get(configuration).delete(path, recursive);
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
index 8b3dbb7..2bfef89 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -36,12 +36,16 @@ import java.util.Map;
@Service
public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
- @Autowired
private ObjectMapper objectMapper;
- @Autowired
private CuratorFramework client;
+ @Autowired
+ public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+ this.objectMapper = objectMapper;
+ this.client = client;
+ }
+
@Override
public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
index ab46418..9f984e0 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java
@@ -38,12 +38,16 @@ import java.util.Map;
@Service
public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService {
- @Autowired
private ObjectMapper objectMapper;
- @Autowired
private CuratorFramework client;
+ @Autowired
+ public SensorIndexingConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) {
+ this.objectMapper = objectMapper;
+ this.client = client;
+ }
+
@Override
public Map<String, Object> save(String name, Map<String, Object> sensorIndexingConfig) throws RestException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index cb88708..eddfc8d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,10 +17,8 @@
*/
package org.apache.metron.rest.service.impl;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
@@ -28,20 +26,15 @@ import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
-import org.apache.metron.rest.service.HdfsService;
+import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
import org.reflections.Reflections;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.env.Environment;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,72 +42,40 @@ import java.util.Map;
import java.util.Set;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-import static org.apache.metron.rest.MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATH_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_PATTERN_LABEL_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_STATEMENT_KEY;
-import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
@Service
public class SensorParserConfigServiceImpl implements SensorParserConfigService {
- @Autowired
- private Environment environment;
-
- @Autowired
private ObjectMapper objectMapper;
private CuratorFramework client;
+ private GrokService grokService;
+
@Autowired
- public void setClient(CuratorFramework client) {
+ public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) {
+ this.objectMapper = objectMapper;
this.client = client;
+ this.grokService = grokService;
}
- @Autowired
- private HdfsService hdfsService;
-
private Map<String, String> availableParsers;
@Override
public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
- String serializedConfig;
- if (isGrokConfig(sensorParserConfig)) {
- addGrokPathToConfig(sensorParserConfig);
- sensorParserConfig.getParserConfig().putIfAbsent(MetronRestConstants.GROK_PATTERN_LABEL_KEY, sensorParserConfig.getSensorTopic().toUpperCase());
- String statement = (String) sensorParserConfig.getParserConfig().remove(MetronRestConstants.GROK_STATEMENT_KEY);
- serializedConfig = serialize(sensorParserConfig);
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_STATEMENT_KEY, statement);
- saveGrokStatement(sensorParserConfig);
- } else {
- serializedConfig = serialize(sensorParserConfig);
- }
try {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), serializedConfig.getBytes(), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
} catch (Exception e) {
throw new RestException(e);
}
return sensorParserConfig;
}
- private String serialize(SensorParserConfig sensorParserConfig) throws RestException {
- String serializedConfig;
- try {
- serializedConfig = objectMapper.writeValueAsString(sensorParserConfig);
- } catch (JsonProcessingException e) {
- throw new RestException("Could not serialize SensorParserConfig", "Could not serialize " + sensorParserConfig.toString(), e.getCause());
- }
- return serializedConfig;
- }
-
@Override
public SensorParserConfig findOne(String name) throws RestException {
SensorParserConfig sensorParserConfig;
try {
sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client);
- if (isGrokConfig(sensorParserConfig)) {
- addGrokStatementToConfig(sensorParserConfig);
- }
} catch (KeeperException.NoNodeException e) {
return null;
} catch (Exception e) {
@@ -145,7 +106,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
return true;
}
- private List<String> getAllTypes() throws RestException {
+ @Override
+ public List<String> getAllTypes() throws RestException {
List<String> types;
try {
types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot());
@@ -190,20 +152,21 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a parserClassName");
} else {
- MessageParser<JSONObject> parser = null;
+ MessageParser<JSONObject> parser;
try {
parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
+ File temporaryGrokFile = null;
if (isGrokConfig(sensorParserConfig)) {
- saveTemporaryGrokStatement(sensorParserConfig);
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic()).toString());
+ temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic());
+ sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString());
}
parser.configure(sensorParserConfig.getParserConfig());
JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
- if (isGrokConfig(sensorParserConfig)) {
- deleteTemporaryGrokStatement(sensorParserConfig);
+ if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) {
+ temporaryGrokFile.delete();
}
return results;
}
@@ -212,78 +175,4 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
private boolean isGrokConfig(SensorParserConfig sensorParserConfig) {
return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName());
}
-
- private void addGrokStatementToConfig(SensorParserConfig sensorParserConfig) throws RestException {
- String grokStatement = "";
- String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
- if (grokPath != null) {
- String fullGrokStatement = getGrokStatement(grokPath);
- String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
- grokStatement = fullGrokStatement.replaceFirst(patternLabel + " ", "");
- }
- sensorParserConfig.getParserConfig().put(GROK_STATEMENT_KEY, grokStatement);
- }
-
- private void addGrokPathToConfig(SensorParserConfig sensorParserConfig) {
- if (sensorParserConfig.getParserConfig().get(GROK_PATH_KEY) == null) {
- String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
- if (grokStatement != null) {
- sensorParserConfig.getParserConfig().put(GROK_PATH_KEY,
- new Path(environment.getProperty(GROK_DEFAULT_PATH_SPRING_PROPERTY), sensorParserConfig.getSensorTopic()).toString());
- }
- }
- }
-
- private String getGrokStatement(String path) throws RestException {
- try {
- return new String(hdfsService.read(new Path(path)));
- } catch (IOException e) {
- throw new RestException(e);
- }
- }
-
- private void saveGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
- saveGrokStatement(sensorParserConfig, false);
- }
-
- private void saveTemporaryGrokStatement(SensorParserConfig sensorParserConfig) throws RestException {
- saveGrokStatement(sensorParserConfig, true);
- }
-
- private void saveGrokStatement(SensorParserConfig sensorParserConfig, boolean isTemporary) throws RestException {
- String patternLabel = (String) sensorParserConfig.getParserConfig().get(GROK_PATTERN_LABEL_KEY);
- String grokPath = (String) sensorParserConfig.getParserConfig().get(GROK_PATH_KEY);
- String grokStatement = (String) sensorParserConfig.getParserConfig().get(GROK_STATEMENT_KEY);
- if (grokStatement != null) {
- String fullGrokStatement = patternLabel + " " + grokStatement;
- try {
- if (!isTemporary) {
- hdfsService.write(new Path(grokPath), fullGrokStatement.getBytes());
- } else {
- File grokDirectory = new File(getTemporaryGrokRootPath());
- if (!grokDirectory.exists()) {
- grokDirectory.mkdirs();
- }
- FileWriter fileWriter = new FileWriter(new File(grokDirectory, sensorParserConfig.getSensorTopic()));
- fileWriter.write(fullGrokStatement);
- fileWriter.close();
- }
- } catch (IOException e) {
- throw new RestException(e);
- }
- } else {
- throw new RestException("A grokStatement must be provided");
- }
- }
-
- private void deleteTemporaryGrokStatement(SensorParserConfig sensorParserConfig) {
- File file = new File(getTemporaryGrokRootPath(), sensorParserConfig.getSensorTopic());
- file.delete();
- }
-
- private String getTemporaryGrokRootPath() {
- String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
- Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
- return new Path(grokTempPath, authentication.getName()).toString();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
new file mode 100644
index 0000000..f5392be
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctionInfo;
+import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
+import org.apache.metron.common.field.transformation.FieldTransformations;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.rest.model.StellarFunctionDescription;
+import org.apache.metron.rest.model.SensorParserContext;
+import org.apache.metron.rest.service.StellarService;
+import org.json.simple.JSONObject;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Service
+public class StellarServiceImpl implements StellarService {
+
+ @Override
+ public Map<String, Boolean> validateRules(List<String> rules) {
+ Map<String, Boolean> results = new HashMap<>();
+ StellarProcessor stellarProcessor = new StellarProcessor();
+ for(String rule: rules) {
+ try {
+ boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT());
+ results.put(rule, result);
+ } catch (ParseException e) {
+ results.put(rule, false);
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) {
+ JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData());
+ sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> {
+ fieldTransformer.transformAndUpdate(sampleJson, sensorParserContext.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT());
+ }
+ );
+ return sampleJson;
+ }
+
+ @Override
+ public FieldTransformations[] getTransformations() {
+ return FieldTransformations.values();
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = new ArrayList<>();
+ Iterable<StellarFunctionInfo> stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo();
+ stellarFunctionsInfo.forEach(stellarFunctionInfo -> {
+ stellarFunctionDescriptions.add(new StellarFunctionDescription(
+ stellarFunctionInfo.getName(),
+ stellarFunctionInfo.getDescription(),
+ stellarFunctionInfo.getParams(),
+ stellarFunctionInfo.getReturns()));
+ });
+ return stellarFunctionDescriptions;
+ }
+
+ @Override
+ public List<StellarFunctionDescription> getSimpleStellarFunctions() {
+ List<StellarFunctionDescription> stellarFunctionDescriptions = getStellarFunctions();
+ return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription ->
+ stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e6628499/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
index cb7c449..9bd368f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -33,16 +33,17 @@ public class StormAdminServiceImpl implements StormAdminService {
private StormCLIWrapper stormCLIClientWrapper;
+ private GlobalConfigService globalConfigService;
+
+ private SensorParserConfigService sensorParserConfigService;
+
@Autowired
- public void setStormCLIClientWrapper(StormCLIWrapper stormCLIClientWrapper) {
+ public StormAdminServiceImpl(StormCLIWrapper stormCLIClientWrapper, GlobalConfigService globalConfigService, SensorParserConfigService sensorParserConfigService) {
this.stormCLIClientWrapper = stormCLIClientWrapper;
+ this.globalConfigService = globalConfigService;
+ this.sensorParserConfigService = sensorParserConfigService;
}
- @Autowired
- private GlobalConfigService globalConfigService;
-
- @Autowired
- private SensorParserConfigService sensorParserConfigService;
@Override
public TopologyResponse startParserTopology(String name) throws RestException {