You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2020/09/26 08:50:57 UTC
[incubator-streampipes] 01/03: [STREAMPIPES-246] Remove obsolete
module streampipes-app-file-export
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit b762769fbc64f1c4b45eba41fbc23de33807d794
Author: Dominik Riemer <ri...@fzi.de>
AuthorDate: Thu Sep 24 22:45:41 2020 +0200
[STREAMPIPES-246] Remove obsolete module streampipes-app-file-export
---
pom.xml | 1 -
streampipes-app-file-export/pom.xml | 79 -----
.../app/file/export/ElasticsearchAppData.java | 83 ------
.../app/file/export/ElasticsearchConfig.java | 60 ----
.../app/file/export/api/IElasticsearch.java | 36 ---
.../application/AppFileExportApplication.java | 30 --
.../app/file/export/converter/JsonConverter.java | 65 -----
.../app/file/export/impl/Elasticsearch.java | 320 ---------------------
.../app/file/export/model/IndexInfo.java | 45 ---
streampipes-backend/pom.xml | 5 -
.../backend/StreamPipesBackendApplication.java | 9 -
11 files changed, 733 deletions(-)
diff --git a/pom.xml b/pom.xml
index 5853729..7ca3711 100644
--- a/pom.xml
+++ b/pom.xml
@@ -837,7 +837,6 @@
<module>archetypes/streampipes-archetype-pe-processors-jvm</module>
<module>archetypes/streampipes-archetype-pe-sinks-flink</module>
<module>archetypes/streampipes-archetype-pe-processors-flink</module>
- <module>streampipes-app-file-export</module>
<module>streampipes-backend</module>
<module>streampipes-code-generation</module>
<module>streampipes-commons</module>
diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml
deleted file mode 100644
index 211640b..0000000
--- a/streampipes-app-file-export/pom.xml
+++ /dev/null
@@ -1,79 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>streampipes-parent</artifactId>
- <groupId>org.apache.streampipes</groupId>
- <version>0.68.0-SNAPSHOT</version>
- </parent>
-
- <name>StreamPipes App File Export</name>
- <artifactId>streampipes-app-file-export</artifactId>
- <packaging>jar</packaging>
-
- <dependencies>
- <!-- StreamPipes dependencies -->
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-storage-couchdb</artifactId>
- <version>0.68.0-SNAPSHOT</version>
- </dependency>
-
- <!-- External dependencies -->
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>javax.ws.rs-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>fluent-hc</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.lightcouch</groupId>
- <artifactId>lightcouch</artifactId>
- </dependency>
-
- <!-- dependency convergence -->
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- </dependency>
-
- <!-- Test dependencies -->
-
- </dependencies>
-</project>
\ No newline at end of file
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java
deleted file mode 100644
index 794f50b..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.app.file.export;
-
-public class ElasticsearchAppData {
-
- private String index;
-
- private long timestampFrom;
-
- private long timestampTo;
-
- private String output;
-
- private boolean allData;
-
- public ElasticsearchAppData() {
- }
-
- public ElasticsearchAppData(String index, long timestampFrom, long timeStampTo, String output, boolean allData) {
- this.index = index;
- this.timestampFrom = timestampFrom;
- this.timestampTo = timeStampTo;
- this.output = output;
- this.allData = allData;
- }
-
- public String getIndex() {
- return index;
- }
-
- public void setIndex(String index) {
- this.index = index;
- }
-
- public long getTimestampFrom() {
- return timestampFrom;
- }
-
- public void setTimestampFrom(long timestampFrom) {
- this.timestampFrom = timestampFrom;
- }
-
- public long getTimestampTo() {
- return timestampTo;
- }
-
- public void setTimeStampTo(long timestampTo) {
- this.timestampTo = timestampTo;
- }
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-
- public boolean isAllData() {
- return allData;
- }
-
- public void setAllData(boolean allData) {
- this.allData = allData;
- }
-}
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java
deleted file mode 100644
index 0460e18..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.app.file.export;
-
-import org.apache.streampipes.config.SpConfig;
-
-public enum ElasticsearchConfig {
- INSTANCE;
-
- private SpConfig config;
- private final static String HOST = "host";
- private final static String PORT = "port";
- private final static String PROTOCOL = "protocol";
- private final static String DATA_LOCATION = "data_location";
-
- ElasticsearchConfig() {
- config = SpConfig.getSpConfig("storage/elasticsearch");
-
- config.register(HOST, "elasticsearch", "Hostname for the elasticsearch service");
- config.register(PORT, 9200, "Port for the elasticsearch service");
- config.register(PROTOCOL, "http", "Protocol the elasticsearch service");
- config.register(DATA_LOCATION,"/home/user/", "Folder that stores all the created data blobs");
- }
-
- public String getElasticsearchHost() {
- return config.getString(HOST);
- }
-
- public Integer getElasticsearchPort() {
- return config.getInteger(PORT);
- }
-
- public String getElasticsearchURL() {
- return getElasticsearchProtocol()+ "://" + getElasticsearchHost() + ":" + getElasticsearchPort();
- }
-
- public String getElasticsearchProtocol() {
- return config.getString(PROTOCOL);
- }
-
- public String getDataLocation() {
- return config.getString(DATA_LOCATION);
- }
-}
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java
deleted file mode 100644
index c61f670..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.app.file.export.api;
-
-import org.apache.streampipes.app.file.export.ElasticsearchAppData;
-
-import javax.ws.rs.core.Response;
-
-public interface IElasticsearch {
-
- Response createFiles(ElasticsearchAppData data);
-
- Response getFile(String fileName);
-
- Response deleteFile(String fileName);
-
- Response getEndpoints();
-
-
-}
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java
deleted file mode 100644
index 3168892..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.app.file.export.application;
-
-import org.glassfish.jersey.server.ResourceConfig;
-import org.apache.streampipes.app.file.export.impl.Elasticsearch;
-
-
-public class AppFileExportApplication extends ResourceConfig {
-
- public AppFileExportApplication() {
- register(Elasticsearch.class);
- }
-}
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java
deleted file mode 100644
index 5909d42..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.app.file.export.converter;
-
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.StringJoiner;
-
-public class JsonConverter {
-
- private JsonObject elasticJsonRepresentation;
- private JsonParser jsonParser;
-
- public JsonConverter() {
- this.jsonParser = new JsonParser();
- }
-
- public String getCsvHeader(String elasticJsonRepresentation) {
- JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject();
-
- Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet();
- StringJoiner sj = new StringJoiner(";");
-
- for (Map.Entry<String, JsonElement> entry : elements) {
- sj.add(entry.getKey().toString());
- }
-
- return sj.toString() + "\n";
-
- }
-
- public String convertToCsv(String elasticJsonRepresentation) {
- JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject();
-
- Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet();
- StringJoiner sj = new StringJoiner(";");
-
- for (Map.Entry<String, JsonElement> entry : elements) {
- sj.add(entry.getValue().toString());
- }
-
- return sj.toString() + "\n";
-
- }
-
-}
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java
deleted file mode 100644
index c3fb684..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.app.file.export.impl;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.fluent.Request;
-import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.Scroll;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.lightcouch.CouchDbClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.streampipes.app.file.export.ElasticsearchAppData;
-import org.apache.streampipes.app.file.export.ElasticsearchConfig;
-import org.apache.streampipes.app.file.export.api.IElasticsearch;
-import org.apache.streampipes.app.file.export.converter.JsonConverter;
-import org.apache.streampipes.app.file.export.model.IndexInfo;
-import org.apache.streampipes.storage.couchdb.utils.Utils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-@Path("/v1/elasticsearch")
-public class Elasticsearch implements IElasticsearch {
-
- private static String mainFilePath = ElasticsearchConfig.INSTANCE.getDataLocation();
- private static final List<String> excludedIndices = Collections.singletonList(".kibana");
-
- Logger LOG = LoggerFactory.getLogger(Elasticsearch.class);
-
- @POST
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
- @Path("/file")
- @Override
- public Response createFiles(ElasticsearchAppData data) {
- String index = data.getIndex();
- long timestampFrom = data.getTimestampFrom();
- long timeStampTo = data.getTimestampTo();
- String output = data.getOutput();
- boolean allData = data.isAllData();
-
- try {
- RestHighLevelClient client = getRestHighLevelClient();
-
- final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
- SearchRequest searchRequest = new SearchRequest(index);
- searchRequest.scroll(scroll);
- SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-
- if (!allData) {
- searchSourceBuilder.query(QueryBuilders.rangeQuery("timestamp").from(timestampFrom).to(timeStampTo));
- }
-
- searchRequest.source(searchSourceBuilder);
- SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
- String scrollId = searchResponse.getScrollId();
- SearchHit[] searchHits = searchResponse.getHits().getHits();
-
- //Time created in milli sec, index, from, to
- long timestamp = System.currentTimeMillis();
- String fileName = System.currentTimeMillis() + "-" + index + "-" + timestampFrom + "-" + timeStampTo + "." + output;
- String filePath = mainFilePath + fileName;
- FileOutputStream fileStream = this.getFileStream(filePath);
-
- if(("csv").equals(output)) {
- processCSV(client, fileStream, scrollId, scroll, searchHits);
- } else {
- processJSON(client, fileStream, scrollId, scroll, searchHits);
- }
-
- fileStream.close();
-
- CouchDbClient couchDbClient = getCouchDbClient();
- Map<String, Object> map = new HashMap<>();
- map.put("_id", fileName);
- map.put("fileName", fileName);
- map.put("filePath", filePath);
- map.put("createAt", timestamp);
- map.put("from", timestampFrom);
- map.put("to", timeStampTo);
- couchDbClient.save(map);
-
- LOG.info("Created file: " + fileName);
-
- return Response.ok().build();
-
- } catch (IOException e) {
- e.printStackTrace();
- LOG.error(e.getMessage());
- return Response.status(500).entity(e).build();
- }
- }
-
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- @Path("/file/{fileName}")
- public Response getFile(@PathParam("fileName") String fileName) {
- File file = new File(mainFilePath + fileName);
- if (file.exists()) {
- LOG.info("Downloaded file: " + fileName);
- return Response.ok(file, MediaType.APPLICATION_OCTET_STREAM)
- .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"")
- .build();
- } else {
- LOG.info("Download - File not found");
- return Response.status(404).entity("File not found").build();
- }
- }
-
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- @Path("/indices")
- public Response getIndices() {
- String url = ElasticsearchConfig.INSTANCE.getElasticsearchURL() + "/_cat/indices?v";
- try {
- JsonElement jsonResponse = get(url);
-
- JsonArray response = jsonResponse.getAsJsonArray();
- List<IndexInfo> availableIndices = new ArrayList<>();
- for(int i = 0; i < response.size(); i++) {
- JsonObject object = response.get(i).getAsJsonObject();
- String index = object.get("index").getAsString();
- if (!shouldExclude(index)) {
- Integer documentCount = Integer.parseInt(object.get("docs.count").getAsString());
- availableIndices.add(new IndexInfo(index, documentCount));
- }
- }
- return Response.ok(availableIndices).build();
- } catch (IOException e) {
- e.printStackTrace();
- return Response.serverError().build();
- }
- }
-
- private boolean shouldExclude(String index) {
- return excludedIndices.stream().anyMatch(i -> i.equals(index));
- }
-
- @DELETE
- @Path("/file/{fileName}")
- @Override
- public Response deleteFile(@PathParam("fileName") String fileName) {
- CouchDbClient couchDbClient = getCouchDbClient();
- JsonObject found = couchDbClient.find(JsonObject.class, fileName);
- couchDbClient.remove(found.get("_id").getAsString(), found.get("_rev").getAsString());
- File file = new File(mainFilePath + fileName);
- file.delete();
- LOG.info("Deleted: " + fileName);
-
- return Response.ok().build();
- }
-
- @GET
- @Path("/files")
- @Override
- public Response getEndpoints() {
- CouchDbClient couchDbClient = getCouchDbClient();
- List<JsonObject> endpoints = couchDbClient.view("_all_docs").includeDocs(true).query(JsonObject.class);
- String json = new Gson().toJson(endpoints);
-
- return Response.ok(json).build();
- }
-
- private CouchDbClient getCouchDbClient() {
- return Utils.getCouchDbElasticsearchFilesEndppointClient();
- }
-
- private FileOutputStream getFileStream(String filePath) throws IOException {
- File file = new File(filePath);
- file.getParentFile().mkdirs();
- return new FileOutputStream(filePath);
- }
-
- private JsonElement get(String url) throws IOException {
- String jsonResponse = Request.Get(url)
- .addHeader("accept", "application/json")
- .addHeader("Content-Type", "application/json")
- .execute()
- .returnContent().asString();
- return new JsonParser().parse(jsonResponse);
- }
-
- private RestHighLevelClient getRestHighLevelClient() {
- String host = ElasticsearchConfig.INSTANCE.getElasticsearchHost();
- int port = ElasticsearchConfig.INSTANCE.getElasticsearchPort();
-
- return new RestHighLevelClient(
- RestClient.builder(
- new HttpHost(host, port, "http"))
- .setRequestConfigCallback(
- new RestClientBuilder.RequestConfigCallback() {
- @Override
- public RequestConfig.Builder customizeRequestConfig(
- RequestConfig.Builder requestConfigBuilder) {
- return requestConfigBuilder
- .setConnectTimeout(5000)
- .setSocketTimeout(60000);
- }
- })
- );
-
- }
-
- private void processJSON(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, SearchHit[] searchHits) throws IOException {
- fileStream.write("[".getBytes());
- boolean isFirstElement = true;
- for (SearchHit hit : searchHits) {
- if(!isFirstElement)
- fileStream.write(",".getBytes());
- fileStream.write(hit.getSourceAsString().getBytes());
- isFirstElement = false;
- }
-
- while (searchHits != null && searchHits.length > 0) {
-
- SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
- scrollRequest.scroll(scroll);
- SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
- scrollId = searchResponse.getScrollId();
- searchHits = searchResponse.getHits().getHits();
- for (SearchHit hit : searchHits) {
- fileStream.write(",".getBytes());
- fileStream.write(hit.getSourceAsString().getBytes());
- }
- }
- fileStream.write("]".getBytes());
-
- ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
- clearScrollRequest.addScrollId(scrollId);
- ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
-
-
- }
-
- private void processCSV(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll,
- SearchHit[] searchHits) throws IOException {
- JsonConverter jsonConverter = new JsonConverter();
-
- boolean isFirstElement = true;
- for (SearchHit hit : searchHits) {
- if (isFirstElement)
- fileStream.write(jsonConverter.getCsvHeader(hit.getSourceAsString()).getBytes());
- String response = jsonConverter.convertToCsv(hit.getSourceAsString());
- fileStream.write(response.getBytes());
- isFirstElement = false;
-
- }
-
- while (searchHits != null && searchHits.length > 0) {
-
- SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
- scrollRequest.scroll(scroll);
- SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
- scrollId = searchResponse.getScrollId();
- searchHits = searchResponse.getHits().getHits();
- for (SearchHit hit : searchHits) {
- fileStream.write(jsonConverter.convertToCsv(hit.getSourceAsString()).getBytes());
- }
-
- }
-
- ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
- clearScrollRequest.addScrollId(scrollId);
- ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
-
- }
-
-
-
- }
diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java
deleted file mode 100644
index b23e82b..0000000
--- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.app.file.export.model;
-
-public class IndexInfo {
-
- private String indexName;
- private Integer documentCount;
-
- public IndexInfo(String indexName, Integer documentCount) {
- this.indexName = indexName;
- this.documentCount = documentCount;
- }
-
- public String getIndexName() {
- return indexName;
- }
-
- public void setIndexName(String indexName) {
- this.indexName = indexName;
- }
-
- public Integer getDocumentCount() {
- return documentCount;
- }
-
- public void setDocumentCount(Integer documentCount) {
- this.documentCount = documentCount;
- }
-}
diff --git a/streampipes-backend/pom.xml b/streampipes-backend/pom.xml
index e035bd5..7bd3afd 100644
--- a/streampipes-backend/pom.xml
+++ b/streampipes-backend/pom.xml
@@ -32,11 +32,6 @@
<!-- StreamPipes dependencies -->
<dependency>
<groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-app-file-export</artifactId>
- <version>0.68.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-container-base</artifactId>
<version>0.68.0-SNAPSHOT</version>
<exclusions>
diff --git a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java
index 5c6ddbc..b94c9e1 100644
--- a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java
+++ b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java
@@ -20,18 +20,15 @@ package org.apache.streampipes.backend;
import org.apache.shiro.web.env.EnvironmentLoaderListener;
import org.apache.shiro.web.servlet.OncePerRequestFilter;
import org.apache.shiro.web.servlet.ShiroFilter;
-import org.apache.streampipes.app.file.export.application.AppFileExportApplication;
import org.apache.streampipes.manager.operations.Operations;
import org.apache.streampipes.model.pipeline.PipelineOperationStatus;
import org.apache.streampipes.rest.notifications.NotificationListener;
-import org.glassfish.jersey.servlet.ServletContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.boot.web.servlet.ServletListenerRegistrationBean;
-import org.springframework.boot.web.servlet.ServletRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
@@ -66,12 +63,6 @@ public class StreamPipesBackendApplication {
}
@Bean
- public ServletRegistrationBean appFileExportRegistrationBean() {
- ServletContainer jerseyContainer = new ServletContainer(new AppFileExportApplication());
- return new ServletRegistrationBean<>(jerseyContainer, "/api/apps/*");
- }
-
- @Bean
public FilterRegistrationBean shiroFilterBean() {
FilterRegistrationBean<OncePerRequestFilter> bean = new FilterRegistrationBean<>();
bean.setFilter(new ShiroFilter());