cvat/.github/workflows/main.yml

name: CI
on:
  push:
    branches:
      - 'master'
      - 'develop'
  pull_request:
    types: [edited, ready_for_review, opened, synchronize, reopened]
    paths-ignore:
      - 'site/**'
      - '**/*.md'

env:
  CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify
  CVAT_VERSION: "local"

jobs:
  search_cache:
    if: |
      github.event.pull_request.draft == false &&
      !startsWith(github.event.pull_request.title, '[WIP]') &&
      !startsWith(github.event.pull_request.title, '[Dependent]')      
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.get-sha.outputs.sha}}
    env:
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      REPO: ${{ github.repository }}
    steps:
      - name: Getting SHA with cache from the default branch
        id: get-sha
        run: |
          DEFAULT_BRANCH=$(gh api /repos/$REPO | jq -r '.default_branch')
          for sha in $(gh api "/repos/$REPO/commits?per_page=100&sha=$DEFAULT_BRANCH" | jq -r '.[].sha');
          do
            RUN_status=$(gh api /repos/${REPO}/actions/workflows/cache.yml/runs | \
              jq -r ".workflow_runs[]? | select((.head_sha == \"${sha}\") and (.conclusion == \"success\")) | .status")

            if [[ ${RUN_status} == "completed" ]]; then
              SHA=$sha
              break
            fi
          done

          echo Default branch is ${DEFAULT_BRANCH}
          echo Workflow will try to get cache from commit: ${SHA}

          echo "default_branch=${DEFAULT_BRANCH}" >> $GITHUB_OUTPUT
          echo "sha=${SHA}" >> $GITHUB_OUTPUT          

  build:
    needs: search_cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: CVAT server. Getting cache from the default branch
        uses: actions/cache@v3
        with:
          path: /tmp/cvat_cache_server
          key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }}

      - name: CVAT UI. Getting cache from the default branch
        uses: actions/cache@v3
        with:
          path: /tmp/cvat_cache_ui
          key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Create artifact directories
        run: |
          mkdir /tmp/cvat_server
          mkdir /tmp/cvat_ui
          mkdir /tmp/cvat_sdk          

      - name: CVAT server. Build and push
        uses: docker/build-push-action@v3
        with:
          cache-from: type=local,src=/tmp/cvat_cache_server
          context: .
          file: Dockerfile
          tags: cvat/server
          outputs: type=docker,dest=/tmp/cvat_server/image.tar

      - name: CVAT UI. Build and push
        uses: docker/build-push-action@v3
        with:
          cache-from: type=local,src=/tmp/cvat_cache_ui
          context: .
          file: Dockerfile.ui
          tags: cvat/ui
          outputs: type=docker,dest=/tmp/cvat_ui/image.tar

      - name: CVAT SDK. Build
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
            --entrypoint /bin/bash -u root cvat/server \
            -c 'python manage.py spectacular --file /transfer/schema.yml'
          pip3 install --user -r cvat-sdk/gen/requirements.txt
          cd cvat-sdk/
          gen/generate.sh
          cd ..

          cp -r cvat-sdk/* /tmp/cvat_sdk/          

      - name: Upload CVAT server artifact
        uses: actions/upload-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/image.tar

      - name: Upload CVAT UI artifact
        uses: actions/upload-artifact@v3
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/image.tar

      - name: Upload CVAT SDK artifact
        uses: actions/upload-artifact@v3
        with:
          name: cvat_sdk
          path: /tmp/cvat_sdk/

  rest_api_testing:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v3
        with:
          python-version: '3.8'

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Download CVAT server image
        uses: actions/download-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI images
        uses: actions/download-artifact@v3
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Download CVAT SDK package
        uses: actions/download-artifact@v3
        with:
          name: cvat_sdk
          path: /tmp/cvat_sdk/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar
          docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
          docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
          docker image ls -a          

      - name: Running REST API and SDK tests
        run: |
          pip3 install --user '/tmp/cvat_sdk/[pytorch]'
          pip3 install --user cvat-cli/
          pip3 install --user -r tests/python/requirements.txt
          pytest tests/python/ -s -v          

      - name: Creating a log file from cvat containers
        if: failure()
        env:
          LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
        run: |
            mkdir $LOGS_DIR
            docker logs test_cvat_server_1 > $LOGS_DIR/cvat_server.log
            docker logs test_cvat_worker_default_1 > $LOGS_DIR/cvat_worker_default.log
            docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log            

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v3.1.1
        with:
          name: rest_api_container_logs
          path: "${{ github.workspace }}/rest_api_testing"

  unit_testing:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Download CVAT server image
        uses: actions/download-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Load Docker server image
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
          docker image ls -a          

      - name: Running OPA tests
        run: |
          python cvat/apps/iam/rules/tests/generate_tests.py \
            --output-dir cvat/apps/iam/rules/

          curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static
          chmod +x ./opa
          ./opa test cvat/apps/iam/rules          

      - name: Running unit tests
        env:
          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
          CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
        run: |
          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server

          max_tries=12
          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done

          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
          -c 'python manage.py test cvat/apps -v 2'

          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
          -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'          

      - name: Creating a log file from cvat containers
        if: failure()
        env:
          LOGS_DIR: "${{ github.workspace }}/unit_testing"
        run: |
            mkdir $LOGS_DIR
            docker logs cvat_server > $LOGS_DIR/cvat_server.log
            docker logs cvat_opa 2> $LOGS_DIR/cvat_opa.log            

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v3.1.1
        with:
          name: unit_tests_container_logs
          path: "${{ github.workspace }}/unit_testing"


  e2e_testing:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3',
                'actions_objects', 'actions_objects2', 'actions_users',
                'actions_projects_models', 'actions_organizations', 'canvas3d_functionality',
                'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons']
    steps:
      - uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - uses: actions/setup-node@v3
        with:
            node-version: '16.x'

      - name: Download CVAT server images
        uses: actions/download-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI image
        uses: actions/download-artifact@v3
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar
          docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
          docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
          docker image ls -a          

      - name: Run CVAT instance
        run: |
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            -f components/serverless/docker-compose.serverless.yml \
            -f tests/docker-compose.minio.yml \
            -f tests/docker-compose.file_share.yml up -d          

      - name: Waiting for server
        env:
          API_ABOUT_PAGE: "localhost:8080/api/server/about"
        run: |
          max_tries=60
          status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
          while [[  $status_code != "200" && max_tries -gt 0 ]]
          do
            echo Number of attempts left: $max_tries
            echo Status code of response: $status_code

            sleep 5
            status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
            (( max_tries-- ))
          done          

      - name: Run E2E tests
        env:
          DJANGO_SU_NAME: 'admin'
          DJANGO_SU_EMAIL: 'admin@localhost.company'
          DJANGO_SU_PASSWORD: '12qwaszx'
        run: |
          docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
          cd ./tests
          yarn --frozen-lockfile

          if [[ ${{ matrix.specs }} == canvas3d_* ]]; then
            npx cypress run \
              --headed \
              --browser chrome \
              --env coverage=false \
              --config-file cypress_canvas3d.json \
              --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
          else
            npx cypress run \
              --browser chrome \
              --env coverage=false \
              --spec 'cypress/integration/${{ matrix.specs }}/**/*.js,cypress/integration/remove_users_tasks_projects_organizations.js'
          fi          

      - name: Creating a log file from "cvat" container logs
        if: failure()
        run: |
                        docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log

      - name: Uploading "cvat" container logs as an artifact
        if: failure()
        uses: actions/upload-artifact@v3.1.1
        with:
          name: e2e_container_logs
          path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log

      - name: Uploading cypress screenshots as an artifact
        if: failure()
        uses: actions/upload-artifact@v3.1.1
        with:
          name: cypress_screenshots_${{ matrix.specs }}
          path: ${{ github.workspace }}/tests/cypress/screenshots

  generate_github_pages:
    if: github.ref == 'refs/heads/develop'
    needs: [rest_api_testing, unit_testing, e2e_testing]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0

      - name: Download CVAT server images
        uses: actions/download-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT server images
        uses: actions/download-artifact@v3
        with:
          name: cvat_sdk
          path: /tmp/cvat_sdk/

      - name: Load Docker images
        run: |
                    docker load --input /tmp/cvat_server/image.tar

      - name: Setup Hugo
        uses: peaceiris/actions-hugo@v2
        with:
          hugo-version: '0.83.1'
          extended: true

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: '16.x'

      - name: Install npm packages
        working-directory: ./site
        run: |
                    npm ci

      - name: Build docs
        run: |
          pip install -r site/requirements.txt
          python site/process_sdk_docs.py --input-dir /tmp/cvat_sdk/docs/ --site-root site/
          python site/build_docs.py          

      - name: Deploy
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./public
          force_orphan: true

  publish_dev_images:
    if: github.ref == 'refs/heads/develop'
    needs: [rest_api_testing, unit_testing, e2e_testing, generate_github_pages]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Download CVAT server images
        uses: actions/download-artifact@v3
        with:
          name: cvat_server
          path: /tmp/cvat_server/

      - name: Download CVAT UI images
        uses: actions/download-artifact@v3
        with:
          name: cvat_ui
          path: /tmp/cvat_ui/

      - name: Load Docker images
        run: |
          docker load --input /tmp/cvat_server/image.tar
          docker load --input /tmp/cvat_ui/image.tar          

      - name: Login to Docker Hub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Push to Docker Hub
        env:
          SERVER_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/server
          UI_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/ui
        run: |
          docker tag cvat/server:latest "${SERVER_IMAGE_REPO}:dev"
          docker push "${SERVER_IMAGE_REPO}:dev"

          docker tag cvat/ui:latest "${UI_IMAGE_REPO}:dev"
          docker push "${UI_IMAGE_REPO}:dev"