diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 00000000..c35a9896 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,21 @@ +# Workflow deletes image artifacts that created by CI workflow +name: Delete image artifacts +on: + workflow_run: + workflows: [CI] + types: + - completed + +jobs: + cleanup: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Clean up + run: | + wri=${{ github.event.workflow_run.id }} + for ai in $(gh api /repos/${{ github.repository }}/actions/runs/$wri/artifacts | jq '.artifacts[] | select( .name | startswith("cvat")) | .id'); + do + gh api --method DELETE /repos/${{ github.repository }}/actions/artifacts/$ai + done \ No newline at end of file diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml index 140c3339..0a2b54f5 100644 --- a/.github/workflows/comment.yml +++ b/.github/workflows/comment.yml @@ -1,4 +1,3 @@ -# v0.1.2 name: Comment on: issue_comment: @@ -59,10 +58,6 @@ jobs: uses: ./.github/workflows/full.yml with: ref: ${{ needs.verify_author.outputs.ref }} - secrets: - DOCKERHUB_CI_WORKSPACE: ${{ secrets.DOCKERHUB_CI_WORKSPACE }} - DOCKERHUB_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} - DOCKERHUB_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} send_status: runs-on: ubuntu-latest diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index ad00a129..dcac9dda 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -5,17 +5,8 @@ on: ref: type: string required: true - secrets: - DOCKERHUB_CI_WORKSPACE: - required: true - DOCKERHUB_CI_USERNAME: - required: true - DOCKERHUB_CI_TOKEN: - required: true env: - SERVER_IMAGE_TEST_REPO: cvat_server - UI_IMAGE_TEST_REPO: cvat_ui WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} jobs: @@ -48,18 +39,8 @@ jobs: build: needs: search_cache runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - with: - ref: ${{ inputs.ref }} - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + - uses: actions/checkout@v2 - name: CVAT server. Getting cache from the default branch uses: actions/cache@v3 @@ -67,40 +48,28 @@ jobs: path: /tmp/cvat_cache_server key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }} - - name: Getting CVAT UI cache from the default branch + - name: CVAT UI. Getting cache from the default branch uses: actions/cache@v3 with: path: /tmp/cvat_cache_ui key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master - with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} - - - name: CVAT UI. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master - with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Create image directory + run: | + mkdir /tmp/cvat_server + mkdir /tmp/cvat_ui + - name: CVAT server. Build and push uses: docker/build-push-action@v3 with: cache-from: type=local,src=/tmp/cvat_cache_server context: . file: Dockerfile - push: true - tags: ${{ steps.meta-server.outputs.tags }} - labels: ${{ steps.meta-server.outputs.labels }} + tags: cvat/server + outputs: type=docker,dest=/tmp/cvat_server/image.tar - name: CVAT UI. Build and push uses: docker/build-push-action@v3 @@ -108,12 +77,23 @@ jobs: cache-from: type=local,src=/tmp/cvat_cache_ui context: . file: Dockerfile.ui - push: true - tags: ${{ steps.meta-ui.outputs.tags }} - labels: ${{ steps.meta-ui.outputs.labels }} + tags: cvat/ui + outputs: type=docker,dest=/tmp/cvat_ui/image.tar - unit_testing: - needs: [build, search_cache] + - name: Upload CVAT server artifact + uses: actions/upload-artifact@v3 + with: + name: cvat_server + path: /tmp/cvat_server/image.tar + + - name: Upload CVAT UI artifact + uses: actions/upload-artifact@v3 + with: + name: cvat_ui + path: /tmp/cvat_ui/image.tar + + rest_api: + needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -159,35 +139,69 @@ jobs: load: true build-args: ELK_VERSION=6.8.23 - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master + - name: Download CVAT server image + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} + name: cvat_server + path: /tmp/cvat_server/ - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master + - name: Download CVAT UI images + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} + name: cvat_ui + path: /tmp/cvat_ui/ - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a - - name: Pull CVAT images + - name: Running REST API tests + run: | + pip3 install --user cvat-sdk/ + pip3 install --user -r tests/rest_api/requirements.txt + pytest tests/rest_api/ -s -v + + - name: Creating a log file from cvat containers + if: failure() + env: + LOGS_DIR: "${{ github.workspace }}/rest_api" run: | - docker pull ${{ steps.meta-server.outputs.tags }} - docker tag ${{ steps.meta-server.outputs.tags }} cvat/server + mkdir $LOGS_DIR + docker logs test_cvat_1 > $LOGS_DIR/cvat.log + docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v2 + env: + LOGS_DIR: "${{ github.workspace }}/rest_api" + with: + name: container_logs + path: $LOGS_DIR - docker pull ${{ steps.meta-ui.outputs.tags }} - docker tag ${{ steps.meta-ui.outputs.tags }} cvat/ui + unit_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@master + + - name: Download CVAT server image + uses: actions/download-artifact@v3 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker image ls -a - name: Running OPA tests run: | @@ -195,13 +209,6 @@ jobs: chmod +x ./opa ./opa test cvat/apps/iam/rules - - name: Running REST API tests - run: | - pip3 install --user cvat-sdk/ - pip3 install --user -r tests/rest_api/requirements.txt - pytest tests/rest_api/ -s -v - pytest tests/rest_api/ --stop-services - - name: Running unit tests env: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} @@ -232,11 +239,11 @@ jobs: env: LOGS_DIR: "${{ github.workspace }}/unit_testing" with: - name: cvat_container_logs + name: container_logs path: $LOGS_DIR e2e_testing: - needs: [build, search_cache] + needs: build runs-on: ubuntu-latest strategy: fail-fast: false @@ -257,35 +264,23 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@master - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master + - name: Download CVAT server image + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} + name: cvat_server + path: /tmp/cvat_server/ - - name: CVAT UI. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master + - name: Download CVAT UI image + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} - tags: | - type=raw,value=${{ inputs.ref }} + name: cvat_ui + path: /tmp/cvat_ui/ - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} - - - name: Pull CVAT images + - name: Load Docker images run: | - docker pull ${{ steps.meta-server.outputs.tags }} - docker tag ${{ steps.meta-server.outputs.tags }} cvat/server - - docker pull ${{ steps.meta-ui.outputs.tags }} - docker tag ${{ steps.meta-ui.outputs.tags }} cvat/ui + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a - name: Run CVAT instance run: | @@ -342,7 +337,7 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: cvat_container_logs + name: container_logs path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log - name: Uploading cypress screenshots as an artifact diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4589c2fc..5bf028eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,10 +10,6 @@ on: - 'site/**' - '**/*.md' -env: - SERVER_IMAGE_TEST_REPO: cvat_server - UI_IMAGE_TEST_REPO: cvat_ui - jobs: search_cache: if: | @@ -54,12 +50,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} - - name: CVAT server. Getting cache from the default branch uses: actions/cache@v3 with: @@ -72,30 +62,22 @@ jobs: path: /tmp/cvat_cache_ui key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master - with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} - - - name: CVAT UI. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master - with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Create image directory + run: | + mkdir /tmp/cvat_server + mkdir /tmp/cvat_ui + - name: CVAT server. Build and push uses: docker/build-push-action@v3 with: cache-from: type=local,src=/tmp/cvat_cache_server context: . file: Dockerfile - push: true - tags: ${{ steps.meta-server.outputs.tags }} - labels: ${{ steps.meta-server.outputs.labels }} + tags: cvat/server + outputs: type=docker,dest=/tmp/cvat_server/image.tar - name: CVAT UI. Build and push uses: docker/build-push-action@v3 @@ -103,12 +85,23 @@ jobs: cache-from: type=local,src=/tmp/cvat_cache_ui context: . file: Dockerfile.ui - push: true - tags: ${{ steps.meta-ui.outputs.tags }} - labels: ${{ steps.meta-ui.outputs.labels }} + tags: cvat/ui + outputs: type=docker,dest=/tmp/cvat_ui/image.tar - unit_testing: - needs: [build, search_cache] + - name: Upload CVAT server artifact + uses: actions/upload-artifact@v3 + with: + name: cvat_server + path: /tmp/cvat_server/image.tar + + - name: Upload CVAT UI artifact + uses: actions/upload-artifact@v3 + with: + name: cvat_ui + path: /tmp/cvat_ui/image.tar + + rest_api: + needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -120,31 +113,67 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master + - name: Download CVAT server image + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} + name: cvat_server + path: /tmp/cvat_server/ - - name: CVAT UI. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master + - name: Download CVAT UI images + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} + name: cvat_ui + path: /tmp/cvat_ui/ - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a - - name: Pull CVAT images + - name: Running REST API tests run: | - docker pull ${{ steps.meta-server.outputs.tags }} - docker tag ${{ steps.meta-server.outputs.tags }} cvat/server + pip3 install --user cvat-sdk/ + pip3 install --user -r tests/rest_api/requirements.txt + pytest tests/rest_api/ -k 'GET' -s - docker pull ${{ steps.meta-ui.outputs.tags }} - docker tag ${{ steps.meta-ui.outputs.tags }} cvat/ui + - name: Creating a log file from cvat containers + if: failure() + env: + LOGS_DIR: "${{ github.workspace }}/rest_api" + run: | + mkdir $LOGS_DIR + docker logs test_cvat_1 > $LOGS_DIR/cvat.log + docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v2 + env: + LOGS_DIR: "${{ github.workspace }}/rest_api" + with: + name: container_logs + path: $LOGS_DIR + + unit_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Download CVAT server image + uses: actions/download-artifact@v3 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Load Docker server image + run: | + docker load --input /tmp/cvat_server/image.tar + docker image ls -a - name: Running OPA tests run: | @@ -152,13 +181,6 @@ jobs: chmod +x ./opa ./opa test cvat/apps/iam/rules - - name: Running REST API tests - run: | - pip3 install --user cvat-sdk/ - pip3 install --user -r tests/rest_api/requirements.txt - pytest tests/rest_api/ -k 'GET' -s - pytest tests/rest_api/ --stop-services - - name: Running unit tests env: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} @@ -190,11 +212,11 @@ jobs: env: LOGS_DIR: "${{ github.workspace }}/unit_testing" with: - name: cvat_container_logs + name: container_logs path: $LOGS_DIR e2e_testing: - needs: [build, search_cache] + needs: build runs-on: ubuntu-latest strategy: fail-fast: false @@ -206,35 +228,27 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: CVAT server. Extract metadata (tags, labels) for Docker - id: meta-server - uses: docker/metadata-action@master + - uses: actions/setup-node@v2 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }} + node-version: '16.x' - - name: CVAT UI. Extract metadata (tags, labels) for Docker - id: meta-ui - uses: docker/metadata-action@master + - name: Download CVAT server images + uses: actions/download-artifact@v3 with: - images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }} + name: cvat_server + path: /tmp/cvat_server/ - - name: Login to Docker Hub - uses: docker/login-action@v2 + - name: Download CVAT UI images + uses: actions/download-artifact@v3 with: - username: ${{ secrets.DOCKERHUB_CI_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_TOKEN }} + name: cvat_ui + path: /tmp/cvat_ui/ - - name: Pull CVAT images + - name: Load Docker images run: | - docker pull ${{ steps.meta-server.outputs.tags }} - docker tag ${{ steps.meta-server.outputs.tags }} cvat/server - - docker pull ${{ steps.meta-ui.outputs.tags }} - docker tag ${{ steps.meta-ui.outputs.tags }} cvat/ui - - - uses: actions/setup-node@v2 - with: - node-version: '16.x' + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a - name: Run CVAT instance run: | @@ -286,7 +300,7 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: cvat_container_logs + name: container_logs path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log - name: Uploading cypress screenshots as an artifact