CVAT-UI: measure coverage. (#2196)

* Try measure coverage

* Return first logic to runing a tests and measure coverage.

* Prepare to measure coverage.

Temporary disable run Cypress test via Firefox.

* Disable creating nyc output before implementing the coverage measurement.

* Change plugin for nyc to babel.

Add to eclude folders:
3rdparty
tests

Disable run test via Firefox.

* Update travis.yaml

Co-authored-by: Dmitry Kruchinin <dmitryx.kruchinin@intel.com>

.nycrc

@@ -0,0 +1,19 @@
+{
+    "all": true,
+    "recursive": true,
+    "compact": false,
+    "useInlineSourceMaps": false,
+    "extension": [
+        ".js",
+        ".jsx",
+        ".ts",
+        ".tsx"
+    ],
+    "exclude": [
+        "**/3rdparty/*",
+        "**/tests/*"
+    ],
+    "parser-plugins": [
+        "typescript"
+    ]
+}

.travis.yml

@@ -35,14 +35,26 @@ before_script:
 script:
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm install && cd ../cvat-core && npm install && npm run test && coveralls-lcov -v -n ./reports/coverage/lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
-  # Up all containers
   - docker-compose up -d
   # Create superuser
   - docker exec -it cvat bash -ic "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
-  # Install Cypress and run tests
+  # End-to-end testing
-  - cd ./tests && npm install
+  - if [[ $TRAVIS_EVENT_TYPE == "cron" && $TRAVIS_BRANCH == "develop" ]];
-  - $(npm bin)/cypress run --headless --browser chrome
+    then
-  - $(npm bin)/cypress run --headless --browser firefox && cd ..
+      cd ./tests && npm install &&
+      npm run cypress:run:firefox && cd ..  &&
+      npm install  &&
+      npm run coverage  &&
+      docker-compose up -d --build  &&
+      cd ./tests &&
+      npm run cypress:run:chrome && cd .. &&
+      npm run report:coverage:text &&
+      npm run report:coverage:lcov;
+    else
+      cd ./tests && npm install &&
+      npm run cypress:run:chrome && cd ..;
+    fi;
+
 
 after_success:
   # https://coveralls-python.readthedocs.io/en/latest/usage/multilang.html

package.json

@@ -8,6 +8,11 @@
   },
   "dependencies": {},
   "devDependencies": {
+    "@istanbuljs/nyc-config-babel": "^1.0.1",
+    "babel-plugin-istanbul": "^6.0.0",
+    "babel-register": "^6.26.0",
+    "core-js": "^3.6.5",
+    "coveralls": "^3.1.0",
     "eslint": "^7.3.0",
     "eslint-config-airbnb": "^18.0.1",
     "eslint-plugin-import": "^2.18.2",
@@ -17,6 +22,7 @@
     "eslint-plugin-react": "^7.14.3",
     "eslint-plugin-react-hooks": "^4.0.4",
     "eslint-plugin-security": "^1.4.0",
+    "nyc": "^15.1.0",
     "remark-lint-emphasis-marker": "^2.0.0",
     "remark-lint-list-item-spacing": "^2.0.0",
     "remark-lint-maximum-heading-length": "^2.0.0",
@@ -29,11 +35,18 @@
     "remark-preset-lint-consistent": "^3.0.0",
     "remark-preset-lint-markdown-style-guide": "^3.0.0",
     "remark-preset-lint-recommended": "^4.0.0",
+    "source-map-support": "^0.5.19",
     "stylelint": "^13.6.1",
     "stylelint-config-standard": "^20.0.0"
   },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "coverage": "npm run instrument && npm run cp && npm run rm",
+    "instrument": "nyc instrument cvat-ui cvat-ui_cov && nyc instrument cvat-canvas cvat-canvas_cov && nyc instrument cvat-data cvat-data_cov && nyc instrument cvat-core cvat-core_cov",
+    "cp": "cp -r cvat-ui_cov/* cvat-ui && cp -r cvat-canvas_cov/* cvat-canvas && cp -r cvat-data_cov/* cvat-data && cp -r cvat-core_cov/* cvat-core",
+    "rm": "rm -rf cvat-ui_cov && rm -rf cvat-canvas_cov && rm -rf cvat-data_cov && rm -r cvat-core_cov",
+    "report:coverage:text": "nyc report --reporter=text",
+    "report:coverage:lcov": "nyc report --reporter=text-lcov | coveralls"
   },
   "repository": {
     "type": "git",

tests/cypress/plugins/index.js

@@ -8,8 +8,21 @@
 
 const {imageGenerator} = require('../plugins/imageGenerator/addPlugin')
 const {createZipArchive} = require('../plugins/createZipArchive/addPlugin')
+const istanbul = require('istanbul-lib-coverage')
+const { join } = require('path')
+const { existsSync, mkdirSync, writeFileSync } = require('fs')
+const execa = require('execa')
 
 module.exports = (on) => {
+    let coverageMap = istanbul.createCoverageMap({})
+    const outputFolder = '../.nyc_output'
+    const nycFilename = join(outputFolder, 'out.json')
+
+    if (!existsSync(outputFolder)) {
+      mkdirSync(outputFolder)
+      console.log('created folder %s for output coverage', outputFolder)
+    }
+
     on('task', {imageGenerator})
     on('task', {createZipArchive})
     on('task', {
@@ -18,4 +31,22 @@ module.exports = (on) => {
             return null
         }
     })
+    on('task', {
+        /**
+         * Combines coverage information from single test
+         * with previously collected coverage.
+         */
+        combineCoverage (coverage) {
+          coverageMap.merge(coverage)
+          return null
+        },
+
+        /**
+         * Saves coverage information as a JSON file and calls
+         */
+        coverageReportPrepare () {
+          writeFileSync(nycFilename, JSON.stringify(coverageMap, null, 2))
+          return null
+        }
+    })
 }

tests/cypress/support/index.js

@@ -18,3 +18,19 @@ before(() => {
         })
     }
 })
+
+afterEach(() => {
+    if (Cypress.browser.name === 'chrome') {
+        cy.window().then(win => {
+            if (win.__coverage__) {
+                cy.task('combineCoverage', win.__coverage__)
+            }
+        })
+    }
+})
+
+after(() => {
+    if (Cypress.browser.name === 'chrome') {
+        cy.task('coverageReportPrepare')
+    }
+})

tests/package-lock.json

@@ -1438,6 +1438,12 @@
       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
       "dev": true
     },
+    "istanbul-lib-coverage": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz",
+      "integrity": "sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg==",
+      "dev": true
+    },
     "jimp": {
       "version": "0.14.0",
       "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.14.0.tgz",

tests/package.json

@@ -1,7 +1,12 @@
 {
+  "scripts": {
+    "cypress:run:chrome": "cypress run --headless --browser chrome",
+    "cypress:run:firefox": "cypress run --headless --browser firefox"
+  },
   "devDependencies": {
     "cypress": "^5.0.0",
-    "cypress-file-upload": "^4.1.1"
+    "cypress-file-upload": "^4.1.1",
+    "istanbul-lib-coverage": "^3.0.0"
   },
   "dependencies": {
     "archiver": "^5.0.0",