You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by te...@apache.org on 2020/02/21 14:27:55 UTC
[incubator-streampipes] 02/02: [STREAMPIPES-78] bounding boxes can
now be resized
This is an automated email from the ASF dual-hosted git repository.
tex pushed a commit to branch image-labeling
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 45b3aabd1196d08b79329ea55f8f96a0ca5d8817
Author: tex <te...@fzi.de>
AuthorDate: Fri Feb 21 15:27:37 2020 +0100
[STREAMPIPES-78] bounding boxes can now be resized
---
ui/src/app/core-model/coco/Annotation.ts | 17 ++
.../imageLabeler/helper/reactLabeling.helper.ts | 171 ++++++++++++++++-----
.../imageLabeler/imageLabeler.component.html | 3 +-
.../core-ui/imageLabeler/imageLabeler.component.ts | 110 +++++++------
ui/src/app/core-ui/imageLabeler/interactionMode.ts | 2 +-
5 files changed, 216 insertions(+), 87 deletions(-)
diff --git a/ui/src/app/core-model/coco/Annotation.ts b/ui/src/app/core-model/coco/Annotation.ts
index 60ef7b7..8dd6541 100644
--- a/ui/src/app/core-model/coco/Annotation.ts
+++ b/ui/src/app/core-model/coco/Annotation.ts
@@ -6,4 +6,21 @@ export class Annotation {
area: number;
bbox: [number,number,number,number]; //[x,y,width,height]
iscrowd: number; //(iscrowd=0 in which case polygons are used) or a collection of objects (iscrowd=1 in which case RLE is used)
+
+ //For UI
+ isSelected: boolean = false;
+ isHovered: boolean = false;
+
+ checkIfInArea(cords): boolean {
+ if (this.bbox !== undefined) {
+ if (this.bbox[0] <= cords[0] && cords[0] <= this.bbox[0] + this.bbox[2] &&
+ this.bbox[1] <= cords[1] && cords[1] <= this.bbox[1] + this.bbox[3]
+ ) {
+ return true;
+ }
+ }
+ return false;
+ //TODO Polygon
+ }
+
}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/helper/reactLabeling.helper.ts b/ui/src/app/core-ui/imageLabeler/helper/reactLabeling.helper.ts
index 5c3055b..62bb9a2 100644
--- a/ui/src/app/core-ui/imageLabeler/helper/reactLabeling.helper.ts
+++ b/ui/src/app/core-ui/imageLabeler/helper/reactLabeling.helper.ts
@@ -18,74 +18,161 @@
export class ReactLabelingHelper {
- private static backgroundAlpha = 0.6;
+ private static backgroundHoverAlpha = 0.6;
+ private static backgroundAlpha = 0.2;
//mouse position
- private static lastMouseX = 0;
- private static lastMouseY = 0;
- private static lastMouseXTransformed = 0;
- private static lastMouseYTransformed = 0;
+ private static lastImageCordX = 0;
+ private static lastImageCordY = 0;
+
private static isMouseDown = false;
private static reactHeight = 0;
private static reactWidth = 0;
- static mouseDown(mousePos, mousePosTransformed) {
- this.lastMouseX = mousePos[0];
- this.lastMouseY = mousePos[1];
- this.lastMouseXTransformed = mousePosTransformed[0];
- this.lastMouseYTransformed = mousePosTransformed[1];
+ //resize
+ private static pressedResizeTopLeft = false;
+ private static pressedResizeTopRight = false;
+ private static pressedResizeButtonLeft = false;
+ private static pressedResizeButtonRight = false;
+
+ static mouseDownCreate(imageCord) {
+ this.lastImageCordX = imageCord[0];
+ this.lastImageCordY = imageCord[1];
this.isMouseDown = true;
}
- static mouseMove(mousePos, mousePosTransformed, context, label, color) {
- let mouseX = mousePos[0];
- let mouseY = mousePos[1];
+ static mouseDownResize(imageCord, annotation, scale) {
+ //check if pressing movement button
+ this.lastImageCordX = imageCord[0];
+ this.lastImageCordY = imageCord[1];
- if(this.isMouseDown) {
- this.reactWidth = mouseX - this.lastMouseX;
- this.reactHeight = mouseY - this.lastMouseY;
+ this.pressedResizeTopLeft = false;
+ this.pressedResizeTopRight = false;
+ this.pressedResizeButtonLeft = false;
+ this.pressedResizeButtonRight = false;
- context.globalAlpha = this.backgroundAlpha;
- this.drawBox(this.lastMouseX, this.lastMouseY, this.reactWidth, this.reactHeight, color, label, context, true);
- context.globalAlpha = 1;
- }
+ this.setSelectedResizeBox(annotation, scale);
}
- static mouseUp(mousePos, mousePosTransformed, coco, labelId) {
- this.isMouseDown = false;
- let reactWidth = mousePosTransformed[0] - this.lastMouseXTransformed;
- let reactHeight = mousePosTransformed[1] - this.lastMouseYTransformed;
+ static mouseMoveCreate(imageCord, imageXShift, imageYShift, context, label, color) {
+ if(this.isMouseDown) {
+ this.reactWidth = imageCord[0] - this.lastImageCordX;
+ this.reactHeight = imageCord[1] - this.lastImageCordY;
- coco.addReactAnnotation(this.lastMouseXTransformed, this.lastMouseYTransformed , reactWidth, reactHeight, labelId);
- console.log('Add react Label:', this.lastMouseXTransformed, this.lastMouseYTransformed, reactWidth, reactHeight, labelId)
+ context.strokeStyle = color;
+ context.fillStyle = color;
+ context.beginPath();
+ context.globalAlpha = this.backgroundHoverAlpha;
+ context.fillRect(this.lastImageCordX + imageXShift, this.lastImageCordY + imageYShift,
+ this.reactWidth, this.reactHeight);
+ context.globalAlpha = 1;
+ context.strokeRect(this.lastImageCordX + imageXShift, this.lastImageCordY + imageYShift,
+ this.reactWidth, this.reactHeight);
+ context.fillText(label, this.lastImageCordX + imageXShift, this.lastImageCordY,- 5 );
+ }
}
- static draw(annotation,label, context, color, imageXShift, imageYShift) {
- context.strokeStyle = color;
- context.fillStyle = color;
- let bbox = annotation.bbox;
- this.drawBox(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3], color, label, context, false);
+ static mouseMoveResize(imageCord, annotation) {
+ if (this.pressedResizeTopLeft) {
+ annotation.bbox[2] += annotation.bbox[0]- imageCord[0];
+ annotation.bbox[3] += annotation.bbox[1]- imageCord[1];
+ annotation.bbox[0] = imageCord[0];
+ annotation.bbox[1] = imageCord[1];
+ } else if (this.pressedResizeTopRight) {
+ annotation.bbox[2] = Math.abs(annotation.bbox[0]- imageCord[0]);
+ annotation.bbox[3] += annotation.bbox[1] - imageCord[1];
+ annotation.bbox[1] = imageCord[1];
+ } else if (this.pressedResizeButtonLeft) {
+ annotation.bbox[2] += annotation.bbox[0]- imageCord[0];
+ annotation.bbox[3] = Math.abs(annotation.bbox[1] - imageCord[1]);
+ annotation.bbox[0] = imageCord[0];
+ } else if (this.pressedResizeButtonRight) {
+ annotation.bbox[2] = Math.abs(annotation.bbox[0]- imageCord[0]);
+ annotation.bbox[3] = Math.abs(annotation.bbox[1] - imageCord[1]);
+ }
+
}
- static drawHighlighted(annotation, label, context, color, imageXShift, imageYShift) {
- context.globalAlpha = 0.6;
+ static mouseUpCreate(mousePosTransformed, coco, labelId) {
+ this.isMouseDown = false;
+
+ let reactWidth = mousePosTransformed[0] - this.lastImageCordX;
+ let reactHeight = mousePosTransformed[1] - this.lastImageCordY;
+
+ if (reactWidth > 0 && reactHeight > 0) {
+ coco.addReactAnnotation(this.lastImageCordX, this.lastImageCordY , reactWidth, reactHeight, labelId);
+ console.log('Add react Label:', this.lastImageCordX, this.lastImageCordY, reactWidth, reactHeight, labelId)
+ }
+ }
+
+ static draw(annotation, label, context, color, imageXShift, imageYShift, scale) {
let bbox = annotation.bbox;
- this.drawBox(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3], color, label, context, true);
- context.globalAlpha = 1;
- }
+ // this.drawBox(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3], label, color, context, annotation.isHovered);
- private static drawBox(x,y, widht, height, color, label, context, filled) {
+ if (annotation.isHovered) {
+ context.globalAlpha = this.backgroundHoverAlpha
+ } else if (annotation.isSelected) {
+ context.globalAlpha = this.backgroundHoverAlpha;
+ } else {
+ context.globalAlpha = this.backgroundAlpha;
+ }
context.strokeStyle = color;
context.fillStyle = color;
context.beginPath();
- if (filled) {
- context.fillRect(x, y, widht, height);
- } else {
- context.rect(x, y, widht, height);
+ context.fillRect(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3]);
+ context.globalAlpha = 1;
+ context.strokeRect(bbox[0] + imageXShift, bbox[1] + imageYShift, bbox[2], bbox[3]);
+ context.font = '12px Arial';
+ context.fillText(label, bbox[0] + imageXShift, bbox[1] + imageYShift - 5 );
+ //context.stroke();
+
+ if (annotation.isSelected) {
+ let size = this.getResizeBoxSize(scale);
+ this.drawCircle(bbox[0] + imageXShift, bbox[1] + imageYShift, size, context);
+ this.drawCircle(bbox[0] + bbox[2] + imageXShift - size, bbox[1] + imageYShift, size, context);
+ this.drawCircle(bbox[0] + imageXShift, bbox[1] + bbox[3] + imageYShift - size, size, context);
+ this.drawCircle( bbox[0] + bbox[2] + imageXShift - size, bbox[1] + bbox[3] + imageYShift - size, size, context);
}
- context.fillText(label, x, y - 5 );
+
+ }
+
+ private static getResizeBoxSize(scale) {
+ return Math.floor(15 / scale);
+ }
+
+ private static drawCircle(x, y, size, context) {
+ context.fillStyle = '#fafafa';
+
+ context.beginPath();
+ context.fillRect(x, y, size, size);
+ context.strokeStyle = 'black';
+ context.strokeRect(x, y, size, size);
context.stroke();
}
+ private static setSelectedResizeBox(annotation, scale) {
+ let size = this.getResizeBoxSize(scale);
+
+ if (annotation.bbox[0] <= this.lastImageCordX && this.lastImageCordX <= annotation.bbox[0] + size &&
+ annotation.bbox[1] <= this.lastImageCordY && this.lastImageCordY <= annotation.bbox[1] + size) {
+ this.pressedResizeTopLeft = true;
+ return true;
+ } else if (annotation.bbox[0] + annotation.bbox[2] - size <= this.lastImageCordX && this.lastImageCordX <= annotation.bbox[0] + annotation.bbox[2] &&
+ annotation.bbox[1] <= this.lastImageCordY && this.lastImageCordY <= annotation.bbox[1] + size) {
+ this.pressedResizeTopRight = true;
+ return true;
+ } else if (annotation.bbox[0] <= this.lastImageCordX && this.lastImageCordX <= annotation.bbox[0] + size &&
+ annotation.bbox[1] + annotation.bbox[3] >= this.lastImageCordY && this.lastImageCordY >= annotation.bbox[1] + annotation.bbox[3] - size) {
+ this.pressedResizeButtonLeft = true;
+ return true;
+ } else if (annotation.bbox[0] + annotation.bbox[2] - size <= this.lastImageCordX && this.lastImageCordX <= annotation.bbox[0] + annotation.bbox[2] &&
+ annotation.bbox[1] + annotation.bbox[3] >= this.lastImageCordY && this.lastImageCordY >= annotation.bbox[1] + annotation.bbox[3] - size) {
+ this.pressedResizeButtonRight = true;
+ return true;
+ }
+ return false;
+ }
+
+
}
\ No newline at end of file
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
index cb88346..8d9cd7f 100644
--- a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.html
@@ -67,7 +67,8 @@
<mat-list>
<mat-list-item *ngFor="let annotation of coco?.annotations" (mouseover)="enterAnnotation(annotation)" (mouseout)="leaveAnnotation(annotation)"
- style="height: 30px; padding-top: 5px; padding-buttom: 5px; border-radius: 50px; margin-bottom: 5px">
+ style="height: 30px; padding-top: 5px; padding-buttom: 5px; border-radius: 50px; margin-bottom: 5px"
+ [style.background-color]="annotation.isHovered || annotation.isSelected ? 'lightgrey' : 'white'">
<mat-icon matListIcon [style.color]="getColor(coco.getLabelById(annotation.category_id))" style="margin-top: -8px;">color_lens</mat-icon>
<h4 style="margin-top: -3px;"> {{annotation.id}} - {{coco.getLabelById(annotation.category_id)}}</h4>
<button mat-icon-button (click)="deleteAnnotation(annotation)" style="margin-top: -10px; margin-left: 10px"> <mat-icon>delete_forever</mat-icon></button>
diff --git a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
index 0257f8e..6c570bd 100644
--- a/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
+++ b/ui/src/app/core-ui/imageLabeler/imageLabeler.component.ts
@@ -59,6 +59,8 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
//scale
private scale: number = 1;
+ private selectedAnnotation;
+
constructor(private restService: DatalakeRestService) {
}
@@ -76,9 +78,6 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
this.isHoverCanvas = false;
}
-
-
-
ngAfterViewInit() {
this.canvas = this.canvasRef.nativeElement;
this.context = this.canvas.getContext('2d');
@@ -105,14 +104,31 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
}
imageMouseDown(e) {
-
if (e.which == 1) {
//left click
+
+ //check if Annotation is click or not and set interaction Mode
+ this.selectedAnnotation = undefined;
+ for(let annotation of this.coco.annotations) {
+ let clicked = annotation.checkIfInArea(this.getImageCords(e.clientX, e.clientY));
+ if (clicked) {
+ annotation.isHovered = false;
+ this.selectedAnnotation = annotation;
+ }
+ annotation.isSelected = clicked;
+ }
+ if (this.selectedAnnotation !== undefined) {
+ this.interactionMode = InteractionMode.ReactResize;
+ } else {
+ this.interactionMode = InteractionMode.ReactLabeling;
+ }
+
if (this.interactionMode == InteractionMode.ReactLabeling) {
- ReactLabelingHelper.mouseDown(this.getMousePosScreen(e.clientX, e.clientY),
- this.getMousePosTransformed(e.clientX, e.clientY));
- } else if (this.interactionMode == InteractionMode.Translate) {
- ImageTranslationHelper.mouseDown(this.getMousePosScreen(e.clientX, e.clientY), this.imageTranslationX, this.imageTranslationY)
+ ReactLabelingHelper.mouseDownCreate(this.getImageCords(e.clientX, e.clientY));
+ } else if (this.interactionMode == InteractionMode.ReactResize){
+ this.draw();
+ ReactLabelingHelper.mouseDownResize(this.getImageCords(e.clientX, e.clientY),
+ this.selectedAnnotation, this.scale)
}
this.isLeftMouseDown = true;
@@ -121,7 +137,7 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
} else {
//right click
this.isRightMouseDown = true;
- ImageTranslationHelper.mouseDown(this.getMousePosScreen(e.clientX, e.clientY), this.imageTranslationX, this.imageTranslationY)
+ ImageTranslationHelper.mouseDown(this.getCanvasCords(e.clientX, e.clientY), this.imageTranslationX, this.imageTranslationY);
}
@@ -131,20 +147,32 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
if (this.isLeftMouseDown) {
if (this.interactionMode == InteractionMode.ReactLabeling) {
- this.draw();
- ReactLabelingHelper.mouseMove(this.getMousePosScreen(e.clientX, e.clientY),
- this.getMousePosTransformed(e.clientX, e.clientY), this.context, this.selectedLabel, this.getColor(this.selectedLabel));
- } else if (this.interactionMode == InteractionMode.Translate) {
- let translation = ImageTranslationHelper.mouseMove(this.getMousePosScreen(e.clientX, e.clientY));
- this.imageTranslationX = translation[0];
- this.imageTranslationY = translation[1];
- this.draw();
+ this.startDraw();
+ this.annotationDraw();
+ let imageXShift = (this.canvasWidth - this.image.width) / 2;
+ let imageYShift =(this.canvasHeight - this.image.height) / 2;
+ ReactLabelingHelper.mouseMoveCreate(this.getImageCords(e.clientX, e.clientY), imageXShift, imageYShift,
+ this.context, this.selectedLabel, this.getColor(this.selectedLabel));
+ this.endDraw();
+ } else if (this.interactionMode == InteractionMode.ReactResize){
+ this.startDraw();
+ this.annotationDraw();
+ ReactLabelingHelper.mouseMoveResize(this.getImageCords(e.clientX, e.clientY), this.selectedAnnotation);
+ this.endDraw();
}
} else if (this.isRightMouseDown) {
- let translation = ImageTranslationHelper.mouseMove(this.getMousePosScreen(e.clientX, e.clientY));
+ let translation = ImageTranslationHelper.mouseMove(this.getCanvasCords(e.clientX, e.clientY));
this.imageTranslationX = translation[0];
this.imageTranslationY = translation[1];
this.draw();
+ } else {
+ for(let annotation of this.coco.annotations) {
+ annotation.isHovered = false;
+ if (!annotation.isSelected) {
+ annotation.isHovered = annotation.checkIfInArea(this.getImageCords(e.clientX, e.clientY))
+ }
+ }
+ this.draw();
}
}
@@ -155,8 +183,7 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
let labelId = this.coco.getLabelId(this.selectedLabel, this.labelCategory);
if (this.interactionMode == InteractionMode.ReactLabeling) {
- ReactLabelingHelper.mouseUp(this.getMousePosScreen(e.clientX, e.clientY),
- this.getMousePosTransformed(e.clientX, e.clientY), this.coco, labelId);
+ ReactLabelingHelper.mouseUpCreate(this.getImageCords(e.clientX, e.clientY), this.coco, labelId);
}
this.draw()
}
@@ -166,7 +193,7 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
}
- draw() {
+ startDraw() {
this.context.clearRect(0, 0, this.canvasWidth, this.canvasHeight);
let newWidth = this.canvasWidth * this.scale;
@@ -179,15 +206,19 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
this.context.scale(this.scale, this.scale);
this.context.drawImage(this.image, this.canvasWidth / 2 - this.image.width / 2, this.canvasHeight / 2 - this.image.height / 2);
+ }
+ annotationDraw() {
for(let annotation of this.coco.annotations) {
- //console.log(this.coco.annotations[0].bbox);
let label = this.coco.getLabelById(annotation.category_id);
//TODO if not BBox
- ReactLabelingHelper.draw(annotation, label, this.context, this.getColor(label),
- ((this.canvasWidth - this.image.width) / 2),
- ((this.canvasHeight - this.image.height) / 2))
+ let imageXShift = (this.canvasWidth - this.image.width) / 2;
+ let imageYShift= (this.canvasHeight - this.image.height) / 2;
+ ReactLabelingHelper.draw(annotation, label, this.context, this.getColor(label), imageXShift, imageYShift, this.scale)
}
+ }
+
+ endDraw() {
this.context.restore();
this.context.beginPath();
@@ -201,6 +232,12 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
this.context.stroke();
}
+ draw() {
+ this.startDraw();
+ this.annotationDraw();
+ this.endDraw();
+ }
+
@HostListener('document:keypress', ['$event'])
handleShortCuts(event: KeyboardEvent) {
if (this.isHoverCanvas) {
@@ -264,40 +301,27 @@ export class ImageLabelerComponent implements OnInit, AfterViewInit {
return colour;
}
- getMousePosScreen(clientX, clientY): [any, any] {
+ getCanvasCords(clientX, clientY): [any, any] {
return [
Math.floor(clientX - this.canvas.getBoundingClientRect().left),
Math.floor(clientY - this.canvas.getBoundingClientRect().top),
]
}
- getMousePosTransformed(clientX, clientY): [any, any] {
+ getImageCords(clientX, clientY): [any, any] {
return [
Math.floor(((clientX - this.canvas.getBoundingClientRect().left) / this.scale) - ((this.canvasWidth / this.scale - this.image.width) / 2) - (this.imageTranslationX / this.scale)),
Math.floor(((clientY - this.canvas.getBoundingClientRect().top) / this.scale) - ((this.canvasHeight / this.scale - this.image.height) / 2) - (this.imageTranslationY / this.scale)),
]
}
- setInteractionModeTranslate() {
- this.interactionMode = InteractionMode.Translate;
- }
-
enterAnnotation(annotation) {
- this.context.save();
- let newWidth = this.canvasWidth * this.scale;
- let newHeight = this.canvasHeight * this.scale;
- this.context.translate(-((newWidth - this.canvasWidth) / 2) + this.imageTranslationX,
- -((newHeight - this.canvasHeight) / 2) + this.imageTranslationY);
- this.context.scale(this.scale, this.scale);
-
- let label = this.coco.getLabelById(annotation.category_id);
- ReactLabelingHelper.drawHighlighted(annotation, label, this.context, this.getColor(label),
- ((this.canvasWidth - this.image.width) / 2),
- ((this.canvasHeight - this.image.height) / 2));
- this.context.restore();
+ annotation.isHovered = true;
+ this.draw()
}
leaveAnnotation(annotation) {
+ annotation.isHovered = false;
this.draw()
}
diff --git a/ui/src/app/core-ui/imageLabeler/interactionMode.ts b/ui/src/app/core-ui/imageLabeler/interactionMode.ts
index 1dc8c51..699a667 100644
--- a/ui/src/app/core-ui/imageLabeler/interactionMode.ts
+++ b/ui/src/app/core-ui/imageLabeler/interactionMode.ts
@@ -1,5 +1,5 @@
export enum InteractionMode {
ReactLabeling,
+ ReactResize,
PolygonLabeling,
- Translate,
}
\ No newline at end of file