You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by te...@apache.org on 2020/02/15 14:56:03 UTC
[incubator-streampipes] branch image-labeling updated:
[STREAMPIPES-78] set bouding boxes correct with zooming, refactor code
This is an automated email from the ASF dual-hosted git repository.
tex pushed a commit to branch image-labeling
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/image-labeling by this push:
new 4ebdd83 [STREAMPIPES-78] set bouding boxes correct with zooming, refactor code
4ebdd83 is described below
commit 4ebdd83ddafd0de5325683f017d1ec85788dfbab
Author: tex <te...@fzi.de>
AuthorDate: Sat Feb 15 15:55:44 2020 +0100
[STREAMPIPES-78] set bouding boxes correct with zooming, refactor code
---
ui/src/app/core-model/coco/Coco.format.ts | 8 +-
.../imageLabeler/imageLabeler.component.css | 6 +-
.../imageLabeler/imageLabeler.component.html | 8 +-
.../core-ui/imageLabeler/imageLabeler.component.ts | 130 ++++++++++-----------
ui/src/app/core-ui/imageLabeler/interactionMode.ts | 5 +
.../labelingHelper/reactLabeling.helper.ts | 55 +++++++++
6 files changed, 137 insertions(+), 75 deletions(-)
diff --git a/ui/src/app/core-model/coco/Coco.format.ts b/ui/src/app/core-model/coco/Coco.format.ts
index 4eb11ed..2251678 100644
--- a/ui/src/app/core-model/coco/Coco.format.ts
+++ b/ui/src/app/core-model/coco/Coco.format.ts
@@ -26,11 +26,11 @@ export class CocoFormat {
this.images.push(image)
}
- getCategoryName(id) {
+ getLabelById(id) {
return this.categories.find(elem => elem.id == id).name;
}
- getCategoryId(name, supercategory): number {
+ getLabelId(name, supercategory): number {
let category = this.categories.find(elem => elem.name == name && elem.supercategory == supercategory);
if (category === undefined) {
category = new Category(this.categories.length + 1, name, supercategory);
@@ -39,12 +39,12 @@ export class CocoFormat {
return category.id;
}
- addReactAnnotation(x,y,width,height, categoryId) {
+ addReactAnnotation(x, y, width, height, categoryId) {
let annnotation = new Annotation();
annnotation.id = this.annotations.length + 1;
annnotation.iscrowd = 0;
annnotation.image_id = 1;
- annnotation.bbox = [x,y,width,height];
+ annnotation.bbox = [x, y, width, height];
annnotation.category_id = categoryId;
this.annotations.push(annnotation)
}
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css
index 41ecef0..44ce11d 100644
--- a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.css
@@ -14,4 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*
- */
\ No newline at end of file
+ */
+canvas {
+ background: white;
+ box-shadow: 1px 1px 1px rgba(0, 0, 0, .2);
+}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
index ec060bf..04afe74 100644
--- a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
@@ -16,13 +16,17 @@
~
-->
-<button mat-button (click)="selectLabel('sign')">Sign</button>
-<button mat-button (click)="selectLabel('car')">Car</button>
+<mat-button-toggle-group name="fontStyle" aria-label="Font Style">
+ <mat-button-toggle (click)="selectLabel('sign')">Sign</mat-button-toggle>
+ <mat-button-toggle (click)="selectLabel('car')">Car</mat-button-toggle>
+ <mat-button-toggle (click)="setInteractionModeTranslate()"><mat-icon>zoom_out_map</mat-icon></mat-button-toggle>
+</mat-button-toggle-group>
<canvas (mousedown)="imageMouseDown($event)" (mousemove)="imageMouseMove($event)" (mouseup)="imageMouseUp($event)"
#test style="border:2px solid;" width="800" height="500" ></canvas>
<!-- (scroll)="imageZoom($event)" -->
+
<button mat-button (click)="zoomin()"> <mat-icon>zoom_in</mat-icon></button>
<button mat-button (click)="zoomout()"> <mat-icon>zoom_out</mat-icon></button>
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
index 02d9f5f..1dfbc26 100644
--- a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
@@ -18,6 +18,8 @@
import { Component, OnInit, ViewChild } from "@angular/core";
import { CocoFormat } from "../../core-model/coco/Coco.format";
+import { InteractionMode } from "./interactionMode";
+import { ReactLabelingHelper } from "./labelingHelper/reactLabeling.helper";
@Component({
selector: 'sp-image-labeler',
@@ -30,96 +32,84 @@ export class ImageLabelerComponent implements OnInit {
private canvas;
private context;
- //mouse position
- private last_mousex = 0;
- private last_mousey = 0;
- private last_mousexTransformed = 0;
- private last_mouseyTransformed = 0;
-
- private mousedown = false;
-
- private reactHeight = 0;
- private reactWidth = 0;
+ private isMouseDown = false;
+ //canvas properties
private canvasWidth;
private canvasHeight;
- private source;
+ //image
+ private image;
+ private imageTranslationX = 0;
+ private imageTranslationY = 0;
private coco;
private label = "Car";
+ //actual interaction mode
+ private interactionMode: InteractionMode = InteractionMode.ReactLabeling;
+
//scale
- private startScale = 1;
- private scale: number = this.startScale;
+ private scale: number = 1;
ngOnInit(): void {
-
this.canvas = this.canvasRef.nativeElement;
this.context = this.canvas.getContext('2d');
- this.canvasWidth = this.canvas.widget;
+ this.canvasWidth = this.canvas.width;
this.canvasHeight= this.canvas.height;
+ this.image = new Image();
- this.source = new Image();
-
- this.source.onload = () => {
- this.context.drawImage(this.source, 0, 0,);
- // context.reactWidth = source.reactWidth;
- // context.height = source.height;
- this.coco = new CocoFormat("Test.png", this.source.width, this.source.height);
- //this.context.strokeRect(0, 0, 20, 30);
-
+ this.image.onload = () => {
+ this.coco = new CocoFormat("Test.png", this.image.width, this.image.height);
+ console.log('Image width: ' + this.image.width);
+ console.log('Image height: ' + this.image.height);
+ this.scale = Math.min(1, this.canvasWidth / this.image.width, this.canvasHeight / this.image.height);
+ console.log('Set Scale to: ' + this.scale);
+ this.draw();
};
- this.source.src = 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg';
+ //this.image.src = 'https://cdn.pixabay.com/photo/2017/10/29/21/05/bridge-2900839_1280.jpg';
+ this.image.src = 'https://www.hamburg.de/contentblob/1740056/7308ff64cbb71631d0463f5b6a34471c/data/bild-kohoevedstrasse3.jpg';
+ // this.source.src = 'https://previews.123rf.com/images/sahua/sahua1503/sahua150300006/38262839-autobahn-stra%C3%9Fe-schilder-autos-und-konstruktionen.jpg';
this.context.lineWidth = 2;
}
imageMouseDown(e) {
- let mousePos = this.getMousePosScreen(e.clientX, e.clientY);
- this.last_mousex = mousePos[0];
- this.last_mousey = mousePos[1];
- let mousePosTransformed = this.getMousePosTransformed(e.clientX, e.clientY);
- this.last_mousexTransformed = mousePosTransformed[0];
- this.last_mouseyTransformed = mousePosTransformed[1];
- this.mousedown = true;
- }
-
- imageMouseMove(e) {
- let mousePos = this.getMousePosScreen(e.clientX, e.clientY);
- let mousex = mousePos[0];
- let mousey = mousePos[1];
- if(this.mousedown) {
- this.draw();
+ if (this.interactionMode == InteractionMode.ReactLabeling) {
+ ReactLabelingHelper.mouseDown(this.getMousePosScreen(e.clientX, e.clientY),
+ this.getMousePosTransformed(e.clientX, e.clientY));
+ } else if (this.interactionMode == InteractionMode.Translate) {
- this.reactWidth = mousex - this.last_mousex;
- this.reactHeight = mousey - this.last_mousey;
- this.context.strokeStyle = this.getColor(this.label);
- this.context.beginPath();
- this.context.rect(this.last_mousex, this.last_mousey, this.reactWidth, this.reactHeight);
- this.context.fillText(this.label, this.last_mousex, this.last_mousey + this.reactHeight);
- this.context.stroke();
}
+ this.isMouseDown = true;
}
- imageMouseUp(e) {
- this.mousedown = false;
+ imageMouseMove(e) {
+ if(this.isMouseDown) {
+ if (this.interactionMode == InteractionMode.ReactLabeling) {
+ this.draw();
+ ReactLabelingHelper.mouseMove(this.getMousePosScreen(e.clientX, e.clientY),
+ this.getMousePosTransformed(e.clientX, e.clientY), this.context, this.label, this.getColor(this.label));
+ }
+ }
- let categoryId = this.coco.getCategoryId(this.label, "");
- let mousePosTransformed = this.getMousePosTransformed(e.clientX, e.clientY);
- let mousexTransformed = mousePosTransformed[0];
- let last_mouseyTransformed = mousePosTransformed[1];
+ }
- let reactWidth = mousexTransformed - this.last_mousexTransformed;
- let reactHeight = last_mouseyTransformed - this.last_mouseyTransformed;
+ imageMouseUp(e) {
+ this.isMouseDown = false;
- this.coco.addReactAnnotation(this.last_mousexTransformed, this.last_mouseyTransformed, reactWidth, reactHeight, categoryId);
- console.log(this.last_mousexTransformed, this.last_mouseyTransformed, reactWidth, reactHeight, categoryId);
+ let labelId = this.coco.getLabelId(this.label, "");
+ if (this.interactionMode == InteractionMode.ReactLabeling) {
+ ReactLabelingHelper.mouseUp(this.getMousePosScreen(e.clientX, e.clientY),
+ this.getMousePosTransformed(e.clientX, e.clientY), this.coco, labelId);
+ }
}
draw() {
+ this.context.clearRect(0, 0, this.canvasWidth, this.canvasHeight);
+
let newWidth = this.canvasWidth * this.scale;
let newHeight = this.canvasHeight * this.scale;
@@ -127,18 +117,17 @@ export class ImageLabelerComponent implements OnInit {
this.context.translate(-((newWidth - this.canvasWidth) / 2), -((newHeight - this.canvasHeight) / 2));
this.context.scale(this.scale, this.scale);
- this.context.drawImage(this.source, 0, 0,);
+ this.context.drawImage(this.image, this.canvasWidth / 2 - this.image.width / 2, this.canvasHeight / 2 - this.image.height / 2);
+
+ this.context.lineWidth = 2;
for(let annotation of this.coco.annotations) {
- console.log(this.coco.annotations[0].bbox);
- let label = this.coco.getCategoryName(annotation.category_id);
- this.context.strokeStyle = this.getColor(label);
- this.context.beginPath();
+ //console.log(this.coco.annotations[0].bbox);
+ let label = this.coco.getLabelById(annotation.category_id);
//TODO if not BBox
- let bbox = annotation.bbox;
- this.context.rect(bbox[0], bbox[1], bbox[2], bbox[3]);
- this.context.fillText(label, bbox[0], bbox[1] + bbox[3]);
- this.context.stroke();
+ ReactLabelingHelper.draw(annotation, label, this.context, this.getColor(label),
+ (this.canvasWidth - this.image.width) / 2,
+ (this.canvasHeight - this.image.height) / 2)
}
this.context.restore();
}
@@ -168,6 +157,7 @@ export class ImageLabelerComponent implements OnInit {
selectLabel(label) {
this.label = label;
+ this.interactionMode = InteractionMode.ReactLabeling;
}
getColor(label) {
@@ -192,11 +182,15 @@ export class ImageLabelerComponent implements OnInit {
getMousePosTransformed(clientX, clientY): [any, any] {
return [
- parseInt((clientX - this.canvas.getBoundingClientRect().left) / this.scale),
- parseInt((clientY - this.canvas.getBoundingClientRect().top) / this.scale),
+ parseInt(((clientX - this.canvas.getBoundingClientRect().left) / this.scale) - ((this.canvasWidth / this.scale - this.image.width) / 2)),
+ parseInt(((clientY - this.canvas.getBoundingClientRect().top) / this.scale) - ((this.canvasHeight / this.scale - this.image.height) / 2)),
]
}
+ setInteractionModeTranslate() {
+ this.interactionMode = InteractionMode.Translate;
+ }
+
}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/interactionMode.ts b/ui/src/app/core-ui/imageLabeler/interactionMode.ts
new file mode 100644
index 0000000..1dc8c51
--- /dev/null
+++ b/ui/src/app/core-ui/imageLabeler/interactionMode.ts
@@ -0,0 +1,5 @@
+export enum InteractionMode {
+ ReactLabeling,
+ PolygonLabeling,
+ Translate,
+}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/labelingHelper/reactLabeling.helper.ts b/ui/src/app/core-ui/imageLabeler/labelingHelper/reactLabeling.helper.ts
new file mode 100644
index 0000000..44b57f7
--- /dev/null
+++ b/ui/src/app/core-ui/imageLabeler/labelingHelper/reactLabeling.helper.ts
@@ -0,0 +1,55 @@
+export class ReactLabelingHelper {
+
+ //mouse position
+ private static lastMouseX = 0;
+ private static lastMouseY = 0;
+ private static lastMouseXTransformed = 0;
+ private static lastMouseYTransformed = 0;
+ private static isMouseDown = false;
+
+ private static reactHeight = 0;
+ private static reactWidth = 0;
+
+ static mouseDown(mousePos, mousePosTransformed) {
+ this.lastMouseX = mousePos[0];
+ this.lastMouseY = mousePos[1];
+ this.lastMouseXTransformed = mousePosTransformed[0];
+ this.lastMouseYTransformed = mousePosTransformed[1];
+ this.isMouseDown = true;
+ console.log(this.lastMouseXTransformed)
+ }
+
+ static mouseMove(mousePos, mousePosTransformed, context, label, color) {
+ let mouseX = mousePos[0];
+ let mouseY = mousePos[1];
+
+ if(this.isMouseDown) {
+ this.reactWidth = mouseX - this.lastMouseX;
+ this.reactHeight = mouseY - this.lastMouseY;
+ context.strokeStyle = color;
+ context.beginPath();
+ context.rect(this.lastMouseX, this.lastMouseY, this.reactWidth, this.reactHeight);
+ context.fillText(label, this.lastMouseX, this.lastMouseY + this.reactHeight);
+ context.stroke();
+ }
+ }
+
+ static mouseUp(mousePos, mousePosTransformed, coco, labelId) {
+ this.isMouseDown = false;
+ let reactWidth = mousePosTransformed[0] - this.lastMouseXTransformed;
+ let reactHeight = mousePosTransformed[1] - this.lastMouseYTransformed;
+
+ coco.addReactAnnotation(this.lastMouseXTransformed, this.lastMouseYTransformed, reactWidth, reactHeight, labelId);
+ console.log('Add react Label:', this.lastMouseXTransformed, this.lastMouseYTransformed, reactWidth, reactHeight, labelId)
+ }
+
+ static draw(annotation,label, context, color, imageXShift, imageYShift) {
+ context.strokeStyle = color;
+ context.beginPath();
+ let bbox = annotation.bbox;
+ context.rect(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3]);
+ context.fillText(label, bbox[0] + imageXShift, bbox[1] + bbox[3] + imageYShift);
+ context.stroke();
+ }
+
+}
\ No newline at end of file