You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by eb...@apache.org on 2020/05/18 11:49:10 UTC
[incubator-streampipes] 01/04: Merge branch 'timeseries-labeling'
into STREAMPIPES-79
This is an automated email from the ASF dual-hosted git repository.
ebi pushed a commit to branch STREAMPIPES-79
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 09d3dc3fa9996f44c9b957bb148464c3ea726293
Merge: dd5fa85 c1c56c9
Author: Daniel Ebi <eb...@fzi.de>
AuthorDate: Thu May 14 09:59:01 2020 +0200
Merge branch 'timeseries-labeling' into STREAMPIPES-79
.../rest/impl/datalake/DataLakeManagementV3.java | 45 +++++-
.../rest/impl/datalake/DataLakeResourceV3.java | 29 +++-
ui/src/app/core-model/datalake/DataResult.ts | 4 +-
.../datalake/datalake-rest.service.ts | 1 +
.../linechart/lineChart.component.ts | 170 ++++++++++++++++-----
.../explorer-widget/explorer.component.ts | 18 ++-
6 files changed, 218 insertions(+), 49 deletions(-)
diff --cc streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeManagementV3.java
index 65786d2,c6ef4ac..a477f05
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeManagementV3.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeManagementV3.java
@@@ -20,9 -20,9 +20,10 @@@ package org.apache.streampipes.rest.imp
import com.google.gson.Gson;
import okhttp3.OkHttpClient;
+import org.apache.commons.io.FileUtils;
import org.influxdb.InfluxDB;
import org.influxdb.InfluxDBFactory;
+ import org.influxdb.dto.Point;
import org.influxdb.dto.Query;
import org.influxdb.dto.QueryResult;
import org.apache.streampipes.config.backend.BackendConfig;
@@@ -523,53 -521,47 +523,96 @@@ public class DataLakeManagementV3
}
}
+
+ public byte[] getImage(String fileRoute) throws IOException {
+ fileRoute = getImageFileRoute(fileRoute);
+ File file = new File(fileRoute);
+ return FileUtils.readFileToByteArray(file);
+ }
+
+
+ public String getImageCoco(String fileRoute) throws IOException {
+ fileRoute = getImageFileRoute(fileRoute);
+ String cocoRoute = getCocoFileRoute(fileRoute);
+
+ File file = new File(cocoRoute);
+ if (!file.exists()) {
+ return "";
+ } else {
+ return FileUtils.readFileToString(file, "UTF-8");
+ }
+ }
+
+
+ public void saveImageCoco(String fileRoute, String data) throws IOException {
+ fileRoute = getImageFileRoute(fileRoute);
+ String cocoRoute = getCocoFileRoute(fileRoute);
+
+ File file = new File(cocoRoute);
+ file.getParentFile().mkdirs();
+ FileUtils.writeStringToFile(file, data, "UTF-8");
+
+ }
+
+ private String getImageFileRoute(String fileRoute) {
+ fileRoute = fileRoute.replace("_", "/");
+ fileRoute = fileRoute.replace("/png", ".png");
+ return fileRoute;
+ }
+
+ private String getCocoFileRoute(String imageRoute) {
+ String[] splitedRoute = imageRoute.split("/");
+ String route = "";
+ for (int i = 0; splitedRoute.length - 2 >= i; i++) {
+ route += "/" + splitedRoute[i];
+ }
+ route += "Coco";
+ route += "/" + splitedRoute[splitedRoute.length - 1];
+ route = route.replace(".png", ".json");
+ return route;
+ }
+
+ public void updateLabels(String index, long startdate, long enddate, String label) {
+ DataResult queryResult = getEvents(index, startdate, enddate);
+ Map<String, String> headerWithTypes = getHeadersWithTypes(index);
+ List<String> headers = queryResult.getHeaders();
+
+ InfluxDB influxDB = getInfluxDBClient();
+ influxDB.setDatabase(BackendConfig.INSTANCE.getInfluxDatabaseName());
+
+ for (List<Object> row : queryResult.getRows()) {
+ long timestampValue = Math.round((double) row.get(headers.indexOf("timestamp")));
+
+ Point.Builder p = Point.measurement(index).time(timestampValue, TimeUnit.MILLISECONDS);
+
+ for (int i = 1; i < row.size(); i++) {
+ String selected_header = headers.get(i);
+ if (!selected_header.equals("sp_internal_label")) {
+ if (headerWithTypes.get(selected_header).equals("integer")) {
+ p.addField(selected_header, Math.round((double) row.get(i)));
+ } else if (headerWithTypes.get(selected_header).equals("string")) {
+ p.addField(selected_header, row.get(i).toString());
+ }
+ } else {
+ p.addField(selected_header, label);
+ }
+ }
+ influxDB.write(p.build());
+ }
+ influxDB.close();
+ }
+
+ private Map<String, String> getHeadersWithTypes(String index) {
+ InfluxDB influxDB = getInfluxDBClient();
+ Query query = new Query("SHOW FIELD KEYS FROM " + index,
+ BackendConfig.INSTANCE.getInfluxDatabaseName());
+ QueryResult result = influxDB.query(query);
+ influxDB.close();
+
+ Map<String, String> headerTypes = new HashMap<String, String>();
+ for (List<Object> element : result.getResults().get(0).getSeries().get(0).getValues()) {
+ headerTypes.put(element.get(0).toString(), element.get(1).toString());
+ }
+ return headerTypes;
+ }
}
diff --cc streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResourceV3.java
index 1828e05,db9b0b7..cc767ea
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResourceV3.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResourceV3.java
@@@ -26,8 -26,8 +26,10 @@@ import org.apache.streampipes.rest.impl
import org.apache.streampipes.rest.impl.datalake.model.GroupedDataResult;
import org.apache.streampipes.rest.impl.datalake.model.PageResult;
import org.apache.streampipes.rest.shared.annotation.GsonWithIds;
+import org.apache.streampipes.rest.shared.annotation.JsonLdSerialized;
+import org.apache.streampipes.rest.shared.util.SpMediaType;
+ import org.influxdb.InfluxDB;
+ import org.influxdb.dto.Point;
import java.io.IOException;
import java.text.ParseException;
@@@ -206,23 -210,19 +212,38 @@@ public class DataLakeResourceV3 extend
.build();
}
+ @GET
+ @Path("/data/image/{route}/file")
+ @Produces("image/png")
+ public Response getImage(@PathParam("route") String fileRoute) throws IOException {
+ return ok(dataLakeManagement.getImage(fileRoute));
+ }
+
++ @POST
++ @Path("/data/image/{route}/coco")
++ public void saveImageCoco(@PathParam("route") String fileRoute, String data) throws IOException {
++ dataLakeManagement.saveImageCoco(fileRoute, data);
++ }
++
+ @GET
+ @Path("/data/image/{route}/coco")
+ @Produces("application/json")
+ public Response getImageCoco(@PathParam("route") String fileRoute) throws IOException {
+ return ok(dataLakeManagement.getImageCoco(fileRoute));
+ }
-
@POST
- @Path("/data/image/{route}/coco")
- public void saveImageCoco(@PathParam("route") String fileRoute, String data) throws IOException {
- dataLakeManagement.saveImageCoco(fileRoute, data);
+ @Produces(MediaType.TEXT_PLAIN)
+ @Path("/data/{index}/{startdate}/{enddate}/labeling")
+ public Response labelData(@Context UriInfo info,
+ @PathParam("index") String index,
+ @PathParam("startdate") long startdate,
+ @PathParam("enddate") long enddate) {
+
+ String label = info.getQueryParameters().getFirst("label");
+ this.dataLakeManagement.updateLabels(index, startdate, enddate, label);
+
+ return Response.ok("Successfully updated database.", MediaType.TEXT_PLAIN).build();
}
+
+
}
diff --cc ui/src/app/core-services/datalake/datalake-rest.service.ts
index 465cc0b,d0e2ccd..6dcefa0
--- a/ui/src/app/core-services/datalake/datalake-rest.service.ts
+++ b/ui/src/app/core-services/datalake/datalake-rest.service.ts
@@@ -103,47 -102,15 +103,48 @@@ export class DatalakeRestService
reportProgress: true,
responseType: 'text'
});
- return this.http.request(request)
+ return this.http.request(request);
}
- saveLabelsInDatabase(index, startDate, endDate, label) {
- const request = new HttpRequest('POST', this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' +
- endDate + '/labeling?label=' + label, {}, {
- reportProgress: true,
- responseType: 'text'
- });
- return this.http.request(request);
+ getImageSrcs() {
+ return [
+ 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2014/04/02/19/32/dead-end-308178_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2015/05/01/14/46/new-york-748595_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2015/02/13/10/18/stop-634941_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2017/04/23/08/43/new-york-2253292_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2015/05/01/14/46/new-york-748595_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2015/02/13/10/18/stop-634941_1280.jpg',
+ 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg',
+ ];
+ }
+
+ getLabels() {
+ return {
+ 'person': ['person', 'Child'],
+ 'vehicle': ['bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat'],
+ 'outdoor': ['traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench'],
+ 'animal': ['bird', 'cat', 'dog'],
+ 'accessory': ['backpack', 'umbrella', 'handbag', 'suitcase'],
+ 'sports': ['frisbee', 'sports ball', 'skis', 'frisbee', 'baseball bat'],
+ 'kitchen': ['bottle', 'cup', 'fork', 'knife', 'spoon'],
+ 'furniture': ['chair', 'couch', 'bed', 'table'],
+ 'electronic': ['tv', 'laptop', 'mouse', 'keyboard']
+ };
+ }
+
+ getImageUrl(imageRoute) {
+ return this.dataLakeUrlV3 + '/data/image/' + imageRoute + '/file';
+ }
+
+ getCocoFileForImage(imageRoute) {
+ return this.http.get(this.dataLakeUrlV3 + '/data/image/' + imageRoute + '/coco');
+ }
+
+ saveCocoFileForImage(imageRoute, data) {
+ return this.http.post(this.dataLakeUrlV3 + '/data/image/' + imageRoute + '/coco', data);
}
-}
++
+}
diff --cc ui/src/app/data-explorer/explorer-widget/explorer.component.ts
index f1e0a4b,20a01e5..0d40513
--- a/ui/src/app/data-explorer/explorer-widget/explorer.component.ts
+++ b/ui/src/app/data-explorer/explorer-widget/explorer.component.ts
@@@ -258,19 -260,26 +259,24 @@@ export class ExplorerComponent implemen
this.selectedInfoResult = this._filter(index)[0];
this.selectedInfoResult.eventSchema.eventProperties.forEach(property => {
- // Check if property is Primitive (only primitives has a runtimeType
+ //Check if property is Primitive (only primitives has a runtimeType)
if (property['runtimeType'] !== undefined) {
if (property['propertyScope'] !== undefined && property['propertyScope'] === 'DIMENSION_PROPERTY') {
- this.dimensionProperties.push(property['runtimeName'])
- }
- //if property is number and is no timestamp property
- else if (this.isNumberProperty(property) &&
+ this.dimensionProperties.push(property['runtimeName']);
+ } else if (this.isNumberProperty(property) &&
(property['domainProperties'] === undefined || (property.domainProperty !== 'http://schema.org/DateTime' &&
property['domainProperties'][0] != 'http://schema.org/DateTime'))) {
this.dataKeys.push(property['runtimeName']);
}
+ else if (this.isLabelProperty(property) &&
+ (property['domainProperties'] === undefined || (property.domainProperty !== 'http://schema.org/DateTime' &&
+ property['domainProperties'][0] != 'http://schema.org/DateTime'))) {
+ this.dataKeys.push(property['runtimeName']);
+ }
} else {
- //list and nested properties
- this.dataKeys.push(property['runtimeName'])
+ // list and nested properties
+ this.dataKeys.push(property['runtimeName']);
}
});
this.selectKey(this.dataKeys.slice(0, 3));