You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by te...@apache.org on 2020/02/14 15:21:14 UTC
[incubator-streampipes] branch image-labeling created (now 84ce65d)
This is an automated email from the ASF dual-hosted git repository.
tex pushed a change to branch image-labeling
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git.
at 84ce65d starting creating image labeler
This branch includes the following new commits:
new 84ce65d starting creating image labeler
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-streampipes] 01/01: starting creating image labeler
Posted by te...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tex pushed a commit to branch image-labeling
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 84ce65d0d9277eeee50490280abf593e0daa2cd1
Author: tex <te...@fzi.de>
AuthorDate: Fri Feb 14 16:20:54 2020 +0100
starting creating image labeler
---
ui/src/app/core-model/coco/Annotation.ts | 9 +
ui/src/app/core-model/coco/Category.ts | 13 ++
ui/src/app/core-model/coco/Coco.format.ts | 52 ++++++
ui/src/app/core-model/coco/Image.ts | 10 +
ui/src/app/core-ui/core-ui.module.ts | 3 +
.../imageLabeler/imageLabeler.component.css | 17 ++
.../imageLabeler/imageLabeler.component.html} | 20 +-
.../core-ui/imageLabeler/imageLabeler.component.ts | 202 +++++++++++++++++++++
.../app/data-explorer/data-explorer.component.html | 9 +-
9 files changed, 323 insertions(+), 12 deletions(-)
diff --git a/ui/src/app/core-model/coco/Annotation.ts b/ui/src/app/core-model/coco/Annotation.ts
new file mode 100644
index 0000000..60ef7b7
--- /dev/null
+++ b/ui/src/app/core-model/coco/Annotation.ts
@@ -0,0 +1,9 @@
+export class Annotation {
+ id: number;
+ image_id: number;
+ category_id: number;
+ segmentation: []; //RLE or [polygon]
+ area: number;
+ bbox: [number,number,number,number]; //[x,y,width,height]
+ iscrowd: number; //(iscrowd=0 in which case polygons are used) or a collection of objects (iscrowd=1 in which case RLE is used)
+}
\ No newline at end of file
diff --git a/ui/src/app/core-model/coco/Category.ts b/ui/src/app/core-model/coco/Category.ts
new file mode 100644
index 0000000..c5a3186
--- /dev/null
+++ b/ui/src/app/core-model/coco/Category.ts
@@ -0,0 +1,13 @@
+export class Category {
+
+
+ constructor(id: number, name: String, supercategory: String) {
+ this.id = id;
+ this.name = name;
+ this.supercategory = supercategory;
+ }
+
+ id: number;
+ name: String;
+ supercategory: String;
+}
\ No newline at end of file
diff --git a/ui/src/app/core-model/coco/Coco.format.ts b/ui/src/app/core-model/coco/Coco.format.ts
new file mode 100644
index 0000000..4eb11ed
--- /dev/null
+++ b/ui/src/app/core-model/coco/Coco.format.ts
@@ -0,0 +1,52 @@
+import { Image } from "./Image";
+import { Annotation } from "./Annotation";
+import { Category } from "./Category";
+
+export class CocoFormat {
+
+
+ info : {
+ "year": number,
+ "version": String,
+ "description": String,
+ "contributor": String,
+ "url": String,
+ "date_created": Date,
+ };
+ licenses: [];
+ images: Image[] = [];
+ annotations: Annotation[] = [];
+ categories: Category[] = [];
+
+ constructor(file_name, width, height) {
+ let image = new Image();
+ image.file_name = file_name;
+ image.height = height;
+ image.width = width;
+ this.images.push(image)
+ }
+
+ getCategoryName(id) {
+ return this.categories.find(elem => elem.id == id).name;
+ }
+
+ getCategoryId(name, supercategory): number {
+ let category = this.categories.find(elem => elem.name == name && elem.supercategory == supercategory);
+ if (category === undefined) {
+ category = new Category(this.categories.length + 1, name, supercategory);
+ this.categories.push(category)
+ }
+ return category.id;
+ }
+
+ addReactAnnotation(x,y,width,height, categoryId) {
+ let annnotation = new Annotation();
+ annnotation.id = this.annotations.length + 1;
+ annnotation.iscrowd = 0;
+ annnotation.image_id = 1;
+ annnotation.bbox = [x,y,width,height];
+ annnotation.category_id = categoryId;
+ this.annotations.push(annnotation)
+ }
+
+}
\ No newline at end of file
diff --git a/ui/src/app/core-model/coco/Image.ts b/ui/src/app/core-model/coco/Image.ts
new file mode 100644
index 0000000..e8f3d2a
--- /dev/null
+++ b/ui/src/app/core-model/coco/Image.ts
@@ -0,0 +1,10 @@
+export class Image {
+ id: Number;
+ width: Number;
+ height: Number;
+ file_name: Date;
+ license: Number;
+ flickr_url: Date;
+ coco_url: Date;
+ date_captured: Date;
+}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/core-ui.module.ts b/ui/src/app/core-ui/core-ui.module.ts
index 275bf4f..082f2d7 100644
--- a/ui/src/app/core-ui/core-ui.module.ts
+++ b/ui/src/app/core-ui/core-ui.module.ts
@@ -29,6 +29,7 @@ import {LineChartComponent} from './linechart/lineChart.component';
import * as PlotlyJS from 'plotly.js/dist/plotly.js';
import { PlotlyModule } from 'angular-plotly.js';
+import { ImageLabelerComponent } from "./imageLabeler/imageLabeler.component";
PlotlyModule.plotlyjs = PlotlyJS;
@NgModule({
@@ -48,6 +49,7 @@ PlotlyModule.plotlyjs = PlotlyJS;
declarations: [
TableComponent,
LineChartComponent,
+ ImageLabelerComponent,
],
providers: [
MatDatepickerModule
@@ -57,6 +59,7 @@ PlotlyModule.plotlyjs = PlotlyJS;
exports: [
TableComponent,
LineChartComponent,
+ ImageLabelerComponent,
]
})
export class CoreUiModule {
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css
new file mode 100644
index 0000000..41ecef0
--- /dev/null
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css
@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
\ No newline at end of file
diff --git a/ui/src/app/data-explorer/data-explorer.component.html b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
similarity index 58%
copy from ui/src/app/data-explorer/data-explorer.component.html
copy to ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
index a7e96bd..ec060bf 100644
--- a/ui/src/app/data-explorer/data-explorer.component.html
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
@@ -16,15 +16,13 @@
~
-->
-<div flex class="page-container page-container-padding" style="height: 1px">
- <div flex="100" layout="column" style="padding:0px;background-color:#f6f6f6; display: inline-block; width: 100%;" >
- <div flex layout-fill layout="row" layout-align="start center"
- style="height:48px;padding-left:10px;font-size:14px;line-height:24px;border-bottom:1px solid #ccc">
- <label style="font-size: 30px; padding-top: 10px;">Data Explorer</label>
- </div>
- </div>
+<button mat-button (click)="selectLabel('sign')">Sign</button>
+<button mat-button (click)="selectLabel('car')">Car</button>
- <div class="container-fluid">
- <sp-explorer></sp-explorer>
- </div>
-</div>
+<canvas (mousedown)="imageMouseDown($event)" (mousemove)="imageMouseMove($event)" (mouseup)="imageMouseUp($event)"
+ #test style="border:2px solid;" width="800" height="500" ></canvas>
+<!-- (scroll)="imageZoom($event)" -->
+
+
+<button mat-button (click)="zoomin()"> <mat-icon>zoom_in</mat-icon></button>
+<button mat-button (click)="zoomout()"> <mat-icon>zoom_out</mat-icon></button>
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
new file mode 100644
index 0000000..02d9f5f
--- /dev/null
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, OnInit, ViewChild } from "@angular/core";
+import { CocoFormat } from "../../core-model/coco/Coco.format";
+
+@Component({
+ selector: 'sp-image-labeler',
+ templateUrl: './imageLabeler.component.html',
+ styleUrls: ['./imageLabeler.component.css']
+})
+export class ImageLabelerComponent implements OnInit {
+
+ @ViewChild('test') canvasRef;
+ private canvas;
+ private context;
+
+ //mouse position
+ private last_mousex = 0;
+ private last_mousey = 0;
+ private last_mousexTransformed = 0;
+ private last_mouseyTransformed = 0;
+
+ private mousedown = false;
+
+ private reactHeight = 0;
+ private reactWidth = 0;
+
+ private canvasWidth;
+ private canvasHeight;
+
+ private source;
+
+ private coco;
+ private label = "Car";
+
+ //scale
+ private startScale = 1;
+ private scale: number = this.startScale;
+
+ ngOnInit(): void {
+
+ this.canvas = this.canvasRef.nativeElement;
+ this.context = this.canvas.getContext('2d');
+ this.canvasWidth = this.canvas.widget;
+ this.canvasHeight= this.canvas.height;
+
+
+ this.source = new Image();
+
+ this.source.onload = () => {
+ this.context.drawImage(this.source, 0, 0,);
+ // context.reactWidth = source.reactWidth;
+ // context.height = source.height;
+ this.coco = new CocoFormat("Test.png", this.source.width, this.source.height);
+ //this.context.strokeRect(0, 0, 20, 30);
+
+ };
+ this.source.src = 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg';
+ this.context.lineWidth = 2;
+ }
+
+ imageMouseDown(e) {
+ let mousePos = this.getMousePosScreen(e.clientX, e.clientY);
+ this.last_mousex = mousePos[0];
+ this.last_mousey = mousePos[1];
+ let mousePosTransformed = this.getMousePosTransformed(e.clientX, e.clientY);
+ this.last_mousexTransformed = mousePosTransformed[0];
+ this.last_mouseyTransformed = mousePosTransformed[1];
+ this.mousedown = true;
+ }
+
+ imageMouseMove(e) {
+ let mousePos = this.getMousePosScreen(e.clientX, e.clientY);
+ let mousex = mousePos[0];
+ let mousey = mousePos[1];
+ if(this.mousedown) {
+ this.draw();
+
+ this.reactWidth = mousex - this.last_mousex;
+ this.reactHeight = mousey - this.last_mousey;
+ this.context.strokeStyle = this.getColor(this.label);
+ this.context.beginPath();
+ this.context.rect(this.last_mousex, this.last_mousey, this.reactWidth, this.reactHeight);
+ this.context.fillText(this.label, this.last_mousex, this.last_mousey + this.reactHeight);
+ this.context.stroke();
+ }
+ }
+
+ imageMouseUp(e) {
+ this.mousedown = false;
+
+ let categoryId = this.coco.getCategoryId(this.label, "");
+
+ let mousePosTransformed = this.getMousePosTransformed(e.clientX, e.clientY);
+ let mousexTransformed = mousePosTransformed[0];
+ let last_mouseyTransformed = mousePosTransformed[1];
+
+ let reactWidth = mousexTransformed - this.last_mousexTransformed;
+ let reactHeight = last_mouseyTransformed - this.last_mouseyTransformed;
+
+ this.coco.addReactAnnotation(this.last_mousexTransformed, this.last_mouseyTransformed, reactWidth, reactHeight, categoryId);
+ console.log(this.last_mousexTransformed, this.last_mouseyTransformed, reactWidth, reactHeight, categoryId);
+
+ }
+
+ draw() {
+ let newWidth = this.canvasWidth * this.scale;
+ let newHeight = this.canvasHeight * this.scale;
+
+ this.context.save();
+ this.context.translate(-((newWidth - this.canvasWidth) / 2), -((newHeight - this.canvasHeight) / 2));
+ this.context.scale(this.scale, this.scale);
+
+ this.context.drawImage(this.source, 0, 0,);
+
+ for(let annotation of this.coco.annotations) {
+ console.log(this.coco.annotations[0].bbox);
+ let label = this.coco.getCategoryName(annotation.category_id);
+ this.context.strokeStyle = this.getColor(label);
+ this.context.beginPath();
+ //TODO if not BBox
+ let bbox = annotation.bbox;
+ this.context.rect(bbox[0], bbox[1], bbox[2], bbox[3]);
+ this.context.fillText(label, bbox[0], bbox[1] + bbox[3]);
+ this.context.stroke();
+ }
+ this.context.restore();
+ }
+
+ imageZoom(e) {
+ console.log(e);
+ let theEvent = e.originalEvent.wheelDelta || e.originalEvent.detail*-1;
+ if(theEvent / 120 > 0) {
+ this.zoomin();
+ } else {
+ this.zoomout();
+ }
+ if (e.preventDefault)
+ e.preventDefault();
+ }
+
+ zoomin()
+ {
+ this.scale += 0.01;
+ this.draw();
+ }
+ zoomout()
+ {
+ this.scale -= 0.01;
+ this.draw();
+ }
+
+ selectLabel(label) {
+ this.label = label;
+ }
+
+ getColor(label) {
+ var hash = 0;
+ for (var i = 0; i < label.length; i++) {
+ hash = label.charCodeAt(i) + ((hash << 5) - hash);
+ }
+ var colour = '#';
+ for (var i = 0; i < 3; i++) {
+ var value = (hash >> (i * 8)) & 0xFF;
+ colour += ('00' + value.toString(16)).substr(-2);
+ }
+ return colour;
+ }
+
+ getMousePosScreen(clientX, clientY): [any, any] {
+ return [
+ parseInt(clientX - this.canvas.getBoundingClientRect().left),
+ parseInt(clientY - this.canvas.getBoundingClientRect().top ),
+ ]
+ }
+
+ getMousePosTransformed(clientX, clientY): [any, any] {
+ return [
+ parseInt((clientX - this.canvas.getBoundingClientRect().left) / this.scale),
+ parseInt((clientY - this.canvas.getBoundingClientRect().top) / this.scale),
+ ]
+ }
+
+
+
+ }
\ No newline at end of file
diff --git a/ui/src/app/data-explorer/data-explorer.component.html b/ui/src/app/data-explorer/data-explorer.component.html
index a7e96bd..339d5b9 100644
--- a/ui/src/app/data-explorer/data-explorer.component.html
+++ b/ui/src/app/data-explorer/data-explorer.component.html
@@ -25,6 +25,13 @@
</div>
<div class="container-fluid">
- <sp-explorer></sp-explorer>
+
+
+ <sp-image-labeler></sp-image-labeler>
+
+
+
+
+ <!-- <sp-explorer></sp-explorer> -->
</div>
</div>