From 6410b86a4edb7fbc3335d624aaf8e13f0afd8f0c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 2 Sep 2022 16:35:23 +0300 Subject: [PATCH] Added end-to-end Cypress tests for Skeletons pipeline (#4892) * Updated config * Added skeleton drawing * Added first cypresst test with skeletons * Reworked split * Added drawing/merging/splitting * Fixed IDs order * Fixed case 107 --- cvat-core/src/annotations-collection.ts | 136 +++++------ cvat-core/src/annotations-objects.ts | 30 +-- tests/cypress.json | 1 + .../cypress/integration/skeletons_pipeline.js | 228 ++++++++++++++++++ tests/cypress/support/commands.js | 66 +++-- tests/docker-compose.file_share.yml | 3 + 6 files changed, 345 insertions(+), 119 deletions(-) create mode 100644 tests/cypress/integration/skeletons_pipeline.js diff --git a/cvat-core/src/annotations-collection.ts b/cvat-core/src/annotations-collection.ts index 53dfadab..5787b994 100644 --- a/cvat-core/src/annotations-collection.ts +++ b/cvat-core/src/annotations-collection.ts @@ -396,50 +396,36 @@ ); } - split(objectState, frame) { - checkObjectType('object state', objectState, null, ObjectState); - checkObjectType('frame', frame, 'integer', null); - - const object = this.objects[objectState.clientID]; - if (typeof object === 'undefined') { - throw new ArgumentError('The object has not been saved yet. Call annotations.put([state]) before'); - } - - if (objectState.objectType !== ObjectType.TRACK) { - return; - } - - const keyframes = Object.keys(object.shapes).sort((a, b) => +a - +b); - if (frame <= +keyframes[0]) { - return; - } - + _splitInternal(objectState, object, frame): ObjectState[] { const labelAttributes = object.label.attributes.reduce((accumulator, attribute) => { accumulator[attribute.id] = attribute; return accumulator; }, {}); - const exported = object.toJSON(); + // first clear all server ids which may exist in the object being splitted + const copy = trackFactory(object.toJSON(), -1, this.injection); + copy.clearServerID(); + const exported = copy.toJSON(); + + // then create two copies, before this frame and after this frame + const prev = { + frame: exported.frame, + group: 0, + label_id: exported.label_id, + attributes: exported.attributes, + shapes: [], + source: Source.MANUAL, + elements: [], + }; + + // after this frame copy is almost the same, except of starting frame + const next = JSON.parse(JSON.stringify(prev)); + next.frame = frame; + + // get position of the object on a frame where user does split and push it to next shape const position = { type: objectState.shapeType, points: objectState.shapeType === ShapeType.SKELETON ? undefined : [...objectState.points], - elements: objectState.shapeType === ShapeType.SKELETON ? objectState.elements.map((el: ObjectState) => { - const elementAttributes = el.attributes; - return { - attributes: Object.keys(elementAttributes).reduce((acc, attrID) => { - acc.push({ - spec_id: +attrID, - value: elementAttributes[attrID], - }); - return acc; - }, []), - label_id: el.label.id, - occluded: el.occluded, - outside: el.outside, - points: [...el.points], - type: el.shapeType, - }; - }) : undefined, rotation: objectState.rotation, occluded: objectState.occluded, outside: objectState.outside, @@ -456,72 +442,66 @@ }, []), frame, }; - - const prev = { - frame: exported.frame, - group: 0, - label_id: exported.label_id, - attributes: exported.attributes, - shapes: [], - source: Source.MANUAL, - }; - - const next = JSON.parse(JSON.stringify(prev)); - next.frame = frame; next.shapes.push(JSON.parse(JSON.stringify(position))); - - exported.shapes.map((shape) => { - delete shape.id; - (shape.elements || []).forEach((element) => { - delete element.id; - }); - + // split all shapes of an initial object into two groups (before/after the frame) + exported.shapes.forEach((shape) => { if (shape.frame < frame) { prev.shapes.push(JSON.parse(JSON.stringify(shape))); } else if (shape.frame > frame) { next.shapes.push(JSON.parse(JSON.stringify(shape))); } + }); + prev.shapes.push(JSON.parse(JSON.stringify(position))); + prev.shapes[prev.shapes.length - 1].outside = true; - return shape; + // do the same recursively for all objet elements if there are any + objectState.elements.forEach((elementState, idx) => { + const elementObject = object.elements[idx]; + const [prevEl, nextEl] = this._splitInternal(elementState, elementObject, frame); + prev.elements.push(prevEl); + next.elements.push(nextEl); }); - prev.shapes.push(position); - // add extra keyframe if no other keyframes before outside - if (!prev.shapes.some((shape) => shape.frame === frame - 1)) { - prev.shapes.push(JSON.parse(JSON.stringify(position))); - prev.shapes[prev.shapes.length - 2].frame -= 1; + return [prev, next]; + } + + split(objectState, frame) { + checkObjectType('object state', objectState, null, ObjectState); + checkObjectType('frame', frame, 'integer', null); + + const object = this.objects[objectState.clientID]; + if (typeof object === 'undefined') { + throw new ArgumentError('The object has not been saved yet. Call annotations.put([state]) before'); } - prev.shapes[prev.shapes.length - 1].outside = true; - (prev.shapes[prev.shapes.length - 1].elements || []).forEach((el) => { - el.outside = true; - }); - let clientID = ++this.count; - const prevTrack = trackFactory(prev, clientID, this.injection); - this.tracks.push(prevTrack); - this.objects[clientID] = prevTrack; + if (objectState.objectType !== ObjectType.TRACK) return; + const keyframes = Object.keys(object.shapes).sort((a, b) => +a - +b); + if (frame <= +keyframes[0]) return; - clientID = ++this.count; - const nextTrack = trackFactory(next, clientID, this.injection); - this.tracks.push(nextTrack); - this.objects[clientID] = nextTrack; + const [prev, next] = this._splitInternal(objectState, object, frame); + const imported = this.import({ + tracks: [prev, next], + tags: [], + shapes: [], + }); // Remove source object object.removed = true; + const [prevImported, nextImported] = imported.tracks; this.history.do( HistoryActions.SPLITTED_TRACK, () => { object.removed = false; - prevTrack.removed = true; - nextTrack.removed = true; + prevImported.removed = true; + nextImported.removed = true; }, () => { object.removed = true; - prevTrack.removed = false; - nextTrack.removed = false; + prevImported.removed = false; + nextImported.removed = false; }, - [object.clientID, prevTrack.clientID, nextTrack.clientID], + [object.clientID, prevImported.clientID, nextImported.clientID], frame, ); } diff --git a/cvat-core/src/annotations-objects.ts b/cvat-core/src/annotations-objects.ts index 9ccc1dd7..bd8b9f03 100644 --- a/cvat-core/src/annotations-objects.ts +++ b/cvat-core/src/annotations-objects.ts @@ -446,7 +446,7 @@ class Annotation { } } - _clearServerID(): void { + clearServerID(): void { this.serverID = undefined; } @@ -509,7 +509,7 @@ class Annotation { public set removed(value: boolean) { if (value) { - this._clearServerID(); + this.clearServerID(); } this._removed = value; } @@ -1162,9 +1162,8 @@ export class Track extends Drawn { } } - _clearServerID(): void { - /* eslint-disable-next-line no-underscore-dangle */ - Drawn.prototype._clearServerID.call(this); + clearServerID(): void { + Drawn.prototype.clearServerID.call(this); for (const keyframe of Object.keys(this.shapes)) { this.shapes[keyframe].serverID = undefined; } @@ -2156,12 +2155,10 @@ export class SkeletonShape extends Shape { } } - _clearServerID(): void { - /* eslint-disable-next-line no-underscore-dangle */ - Shape.prototype._clearServerID.call(this); + clearServerID(): void { + Shape.prototype.clearServerID.call(this); for (const element of this.elements) { - /* eslint-disable-next-line no-underscore-dangle */ - element._clearServerID(); + element.clearServerID(); } } @@ -2722,6 +2719,11 @@ export class SkeletonTrack extends Track { constructor(data: RawTrackData, clientID: number, color: string, injection: AnnotationInjection) { super(data, clientID, color, injection); this.shapeType = ShapeType.SKELETON; + + for (const shape of Object.values(this.shapes)) { + delete shape.points; + } + this.readOnlyFields = ['points', 'label', 'occluded', 'outside']; this.pinned = false; this.elements = data.elements.map((element: RawTrackData['elements'][0]) => ( @@ -2748,12 +2750,10 @@ export class SkeletonTrack extends Track { } } - _clearServerID(): void { - /* eslint-disable-next-line no-underscore-dangle */ - Track.prototype._clearServerID.call(this); + clearServerID(): void { + Track.prototype.clearServerID.call(this); for (const element of this.elements) { - /* eslint-disable-next-line no-underscore-dangle */ - element._clearServerID(); + element.clearServerID(); } } diff --git a/tests/cypress.json b/tests/cypress.json index f521fb1a..6fb01647 100644 --- a/tests/cypress.json +++ b/tests/cypress.json @@ -12,6 +12,7 @@ }, "testFiles": [ "auth_page.js", + "skeletons_pipeline.js", "actions_tasks/**/*.js", "actions_tasks2/**/*.js", "actions_tasks3/**/*.js", diff --git a/tests/cypress/integration/skeletons_pipeline.js b/tests/cypress/integration/skeletons_pipeline.js new file mode 100644 index 00000000..8fff6b38 --- /dev/null +++ b/tests/cypress/integration/skeletons_pipeline.js @@ -0,0 +1,228 @@ +// Copyright (C) 2022 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Manipulations with skeletons', () => { + const skeletonSize = 5; + const labelName = 'skeleton'; + const taskName = 'skeletons main pipeline'; + const imagesFolder = `cypress/fixtures/${taskName}`; + const archiveName = `${taskName}.zip`; + const archivePath = `cypress/fixtures/${archiveName}`; + const imageParams = { + width: 800, + height: 800, + color: 'gray', + textOffset: { x: 10, y: 10 }, + text: 'skeletons pipeline', + count: 5, + }; + let taskID = null; + + before(() => { + cy.visit('auth/login'); + cy.login(); + cy.imageGenerator( + imagesFolder, + taskName, + imageParams.width, + imageParams.height, + imageParams.color, + imageParams.textOffset.x, + imageParams.textOffset.y, + imageParams.text, + imageParams.count, + ); + cy.createZipArchive(imagesFolder, archivePath); + }); + + after(() => { + cy.getAuthKey().then((response) => { + const authKey = response.body.key; + cy.request({ + method: 'DELETE', + url: `/api/tasks/${taskID}`, + headers: { + Authorization: `Token ${authKey}`, + }, + }); + }); + }); + + describe('Create a task with skeletons', () => { + it('Create a simple task', () => { + cy.visit('/tasks/create'); + cy.get('#name').type(taskName); + cy.get('.cvat-constructor-viewer-new-skeleton-item').click(); + cy.get('.cvat-skeleton-configurator').should('exist').and('be.visible'); + + cy.get('.cvat-label-constructor-creator').within(() => { + cy.get('#name').type(labelName); + cy.get('.ant-radio-button-checked').within(() => { + cy.get('.ant-radio-button-input').should('have.attr', 'value', 'point'); + }); + }); + + const pointsOffset = [ + { x: 0.55, y: 0.15 }, + { x: 0.20, y: 0.35 }, + { x: 0.43, y: 0.55 }, + { x: 0.63, y: 0.38 }, + { x: 0.27, y: 0.15 }, + ]; + expect(skeletonSize).to.be.equal(pointsOffset.length); + + cy.get('.cvat-skeleton-configurator-svg').then(($canvas) => { + const canvas = $canvas[0]; + + canvas.scrollIntoView(); + const rect = canvas.getBoundingClientRect(); + const { width, height } = rect; + pointsOffset.forEach(({ x: xOffset, y: yOffset }) => { + canvas.dispatchEvent(new MouseEvent('mousedown', { + clientX: rect.x + width * xOffset, + clientY: rect.y + height * yOffset, + button: 0, + bubbles: true, + })); + }); + + cy.get('.ant-radio-button-wrapper:nth-child(3)').click().within(() => { + cy.get('.ant-radio-button-input').should('have.attr', 'value', 'join'); + }); + + cy.get('.cvat-skeleton-configurator-svg').within(() => { + cy.get('circle').then(($circles) => { + expect($circles.length).to.be.equal(5); + $circles.each(function (i) { + const circle1 = this; + $circles.each(function (j) { + const circle2 = this; + if (i === j) return; + circle1.dispatchEvent(new MouseEvent('mouseover', { bubbles: true })); + circle1.dispatchEvent(new MouseEvent('click', { button: 0, bubbles: true })); + circle1.dispatchEvent(new MouseEvent('mouseout', { bubbles: true })); + + circle2.dispatchEvent(new MouseEvent('mouseover', { bubbles: true })); + circle2.dispatchEvent(new MouseEvent('click', { button: 0, bubbles: true })); + circle2.dispatchEvent(new MouseEvent('mouseout', { bubbles: true })); + }); + }); + }); + }); + + cy.contains('Continue').scrollIntoView().click(); + cy.contains('Continue').scrollIntoView().click(); + cy.get('input[type="file"]').attachFile(archiveName, { subjectType: 'drag-n-drop' }); + + cy.intercept('/api/tasks?**').as('taskPost'); + cy.contains('Submit & Open').scrollIntoView().click(); + + cy.wait('@taskPost').then((interception) => { + taskID = interception.response.body.id; + expect(interception.response.statusCode).to.be.equal(201); + cy.intercept(`/api/tasks/${taskID}?**`).as('getTask'); + cy.wait('@getTask', { timeout: 10000 }); + cy.get('.cvat-task-jobs-table-row').should('exist').and('be.visible'); + cy.openJob(); + }); + }); + }); + }); + + describe('Working with objects', () => { + function createSkeletonObject(shapeType) { + cy.createSkeleton({ + labelName, + xtl: 100, + ytl: 100, + xbr: 300, + ybr: 300, + type: `${shapeType[0].toUpperCase()}${shapeType.slice(1).toLowerCase()}`, + }); + cy.get('#cvat_canvas_shape_1').should('exist').and('be.visible'); + cy.get('#cvat-objects-sidebar-state-item-1').should('exist').and('be.visible') + .within(() => { + cy.get('.cvat-objects-sidebar-state-item-object-type-text').should('have.text', `SKELETON ${shapeType}`.toUpperCase()); + cy.get('.cvat-objects-sidebar-state-item-label-selector').within(() => { + cy.get('input').should('be.disabled'); + }); + cy.get('.cvat-objects-sidebar-state-item-elements-collapse').should('exist').and('be.visible').click(); + cy.get('.cvat-objects-sidebar-state-item-elements').should('have.length', skeletonSize); + }); + } + + function deleteSkeleton(selector, shapeType, force) { + cy.get(selector).trigger('mousemove').should('have.class', 'cvat_canvas_shape_activated'); + cy.get('body').type(force ? '{shift}{del}' : '{del}'); + if (shapeType.toLowerCase() === 'track' && !force) { + cy.get('.cvat-remove-object-confirm-wrapper').should('exist').and('be.visible'); + cy.get('.ant-modal-content').within(() => { + cy.contains('Yes').click(); + }); + } + cy.get(selector).should('not.exist'); + } + + it('Creating and removing a skeleton shape', () => { + createSkeletonObject('shape'); + deleteSkeleton('#cvat_canvas_shape_1', 'shape', false); + cy.removeAnnotations(); + }); + + it('Creating and removing a skeleton track', () => { + createSkeletonObject('track'); + deleteSkeleton('#cvat_canvas_shape_1', 'track', false); + + cy.removeAnnotations(); + + createSkeletonObject('track'); + deleteSkeleton('#cvat_canvas_shape_1', 'track', true); + + cy.removeAnnotations(); + }); + + it('Splitting two skeletons and merge them back', () => { + createSkeletonObject('track'); + + const splittingFrame = Math.trunc(imageParams.count / 2); + cy.goCheckFrameNumber(splittingFrame); + + cy.get('.cvat-split-track-control').click(); + cy.get('#cvat_canvas_shape_1').click().click(); + + // check objects after splitting + cy.get('#cvat_canvas_shape_1').should('not.exist'); + cy.get('#cvat_canvas_shape_18').should('exist').and('not.be.visible'); + cy.get('#cvat_canvas_shape_24').should('exist').and('be.visible'); + + cy.goToNextFrame(splittingFrame + 1); + + cy.get('#cvat_canvas_shape_18').should('not.exist'); + cy.get('#cvat_canvas_shape_24').should('exist').and('be.visible'); + + // now merge them back + cy.get('.cvat-merge-control').click(); + cy.get('#cvat_canvas_shape_24').click(); + + cy.goCheckFrameNumber(0); + + cy.get('#cvat_canvas_shape_18').click(); + cy.get('body').type('m'); + + // and check objects after merge + cy.get('#cvat_canvas_shape_18').should('not.exist'); + cy.get('#cvat_canvas_shape_24').should('not.exist'); + + cy.get('#cvat_canvas_shape_30').should('exist').and('be.visible'); + cy.goCheckFrameNumber(splittingFrame + 1); + cy.get('#cvat_canvas_shape_30').should('exist').and('be.visible'); + cy.goCheckFrameNumber(imageParams.count - 1); + cy.get('#cvat_canvas_shape_30').should('exist').and('be.visible'); + + cy.removeAnnotations(); + }); + }); +}); diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 0144d1b3..be36eb28 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -1,9 +1,12 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corporation // // SPDX-License-Identifier: MIT /// +/* eslint-disable security/detect-non-literal-regexp */ + import { decomposeMatrix } from './utils'; require('cypress-file-upload'); @@ -388,6 +391,22 @@ Cypress.Commands.add('createEllipse', (createEllipseParams) => { cy.checkObjectParameters(createEllipseParams, 'ELLIPSE'); }); +Cypress.Commands.add('createSkeleton', (skeletonParameters) => { + cy.interactControlButton('draw-skeleton'); + cy.switchLabel(skeletonParameters.labelName, 'draw-skeleton'); + cy.get('.cvat-draw-skeleton-popover').within(() => { + cy.get('.ant-select-selection-item').then(($labelValue) => { + selectedValueGlobal = $labelValue.text(); + }); + cy.contains('button', skeletonParameters.type).click(); + }); + cy.get('.cvat-canvas-container') + .click(skeletonParameters.xtl, skeletonParameters.ytl) + .click(skeletonParameters.xbr, skeletonParameters.ybr); + cy.checkPopoverHidden('draw-skeleton'); + cy.checkObjectParameters(skeletonParameters, 'SKELETON'); +}); + Cypress.Commands.add('changeAppearance', (colorBy) => { cy.get('.cvat-appearance-color-by-radio-group').within(() => { cy.get('[type="radio"]').check(colorBy, { force: true }); @@ -868,36 +887,31 @@ Cypress.Commands.add('shapeRotate', (shape, expectedRotateDeg, pressShift = fals .trigger('mousemove') .trigger('mouseover') .should('have.class', 'cvat_canvas_shape_activated'); - cy.get('.svg_select_points_rot').then($el => { - let {x, y, width, height} = $el[0].getBoundingClientRect(); + cy.get('.svg_select_points_rot').then(($el) => { + const rect = $el[0].getBoundingClientRect(); + let { x, y } = rect; + const { width, height } = rect; x += width / 2; y += height / 2; - cy.window().then((win) => { - const [container] = win.document.getElementsByClassName('cvat-canvas-container'); - const {x: offsetX, y: offsetY} = container.getBoundingClientRect(); - // x -= offsetX; - // y -= offsetY; - cy.get('#root') - .trigger('mousemove', x, y) - .trigger('mouseenter', x, y); - cy.get('.svg_select_points_rot').should('have.class', 'cvat_canvas_selected_point'); - cy.get('#root').trigger('mousedown', x, y, { button: 0 }); - if (pressShift) { - cy.get('body').type('{shift}', { release: false }); - } - cy.get('#root').trigger('mousemove', x + 20, y); - cy.get(shape).should('have.attr', 'transform'); - cy.document().then((doc) => { - const modShapeIDString = shape.substring(1); // Remove "#" from the shape id string - const shapeTranformMatrix = decomposeMatrix(doc.getElementById(modShapeIDString).getCTM()); - cy.get('#cvat_canvas_text_content').should('contain.text', `${shapeTranformMatrix}°`); - expect(`${shapeTranformMatrix}°`).to.be.equal(`${expectedRotateDeg}°`); - }); - cy.get('#root').trigger('mouseup'); - }) + cy.get('#root') + .trigger('mousemove', x, y) + .trigger('mouseenter', x, y); + cy.get('.svg_select_points_rot').should('have.class', 'cvat_canvas_selected_point'); + cy.get('#root').trigger('mousedown', x, y, { button: 0 }); + if (pressShift) { + cy.get('body').type('{shift}', { release: false }); + } + cy.get('#root').trigger('mousemove', x + 20, y); + cy.get(shape).should('have.attr', 'transform'); + cy.document().then((doc) => { + const modShapeIDString = shape.substring(1); // Remove "#" from the shape id string + const shapeTranformMatrix = decomposeMatrix(doc.getElementById(modShapeIDString).getCTM()); + cy.get('#cvat_canvas_text_content').should('contain.text', `${shapeTranformMatrix}°`); + expect(`${shapeTranformMatrix}°`).to.be.equal(`${expectedRotateDeg}°`); + }); + cy.get('#root').trigger('mouseup'); }); - }); Cypress.Commands.add('deleteFrame', (action = 'delete') => { diff --git a/tests/docker-compose.file_share.yml b/tests/docker-compose.file_share.yml index 01028163..3d00e023 100644 --- a/tests/docker-compose.file_share.yml +++ b/tests/docker-compose.file_share.yml @@ -4,3 +4,6 @@ services: cvat_worker_default: volumes: - ./tests/cypress/integration/actions_tasks3/assets/case_107:/home/django/share:rw + cvat_server: + volumes: + - ./tests/cypress/integration/actions_tasks3/assets/case_107:/home/django/share:rw