From b5d5f1cbef7a2567377c9ca8b7a343936e0f41a6 Mon Sep 17 00:00:00 2001 From: Nikita Manovich <40690625+nmanovic@users.noreply.github.com> Date: Fri, 7 Aug 2020 06:32:51 +0300 Subject: [PATCH] Siammask tracker as DL serverless function (#1988) * Implement SiamMask serverless function (cannot be deployed yet) * Fix deployment (need to test the function itself). * Removed mask from state in SiamMask serverless function. * Update CHANGELOG.md * Add support of the tracker serverless function into lambda manager. * Add an example how to deploy siammask serverless function. --- CHANGELOG.md | 1 + cvat/apps/lambda_manager/views.py | 7 ++- serverless/deploy.sh | 4 ++ .../foolwood/siammask/nuclio/function.yaml | 11 +++-- .../foolwood/siammask/nuclio/model_handler.py | 44 +++++++++++++++---- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86fa5353..99018445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.1.0] - Unreleased ### Added +- Siammask tracker as DL serverless function () - [Datumaro] Added model info and source info commands () ### Changed diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 43ec2e18..d307ca51 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -155,6 +155,12 @@ class LambdaFunction: payload.update({ "max_distance": max_distance }) + elif self.kind == LambdaType.TRACKER: + payload.update({ + "image": self._get_image(db_task, data["frame"], quality), + "shape": data.get("shape", None), + "state": data.get("state", None) + }) else: raise ValidationError( '`{}` lambda function has incorrect type: {}' @@ -469,7 +475,6 @@ class LambdaJob: LambdaJob._call_reid(function, db_task, quality, kwargs.get("threshold"), kwargs.get("max_distance")) - def return_response(success_code=status.HTTP_200_OK): def wrap_response(func): @wraps(func) diff --git a/serverless/deploy.sh b/serverless/deploy.sh index 74b58519..9b150a02 100755 --- a/serverless/deploy.sh +++ b/serverless/deploy.sh @@ -46,4 +46,8 @@ nuctl deploy --project-name cvat \ --path $SCRIPT_DIR/tensorflow/faster_rcnn_inception_v2_coco/nuclio \ --platform local +nuctl deploy --project-name cvat \ + --path $SCRIPT_DIR/pytorch/foolwood/siammask/nuclio \ + --platform local + nuctl get function diff --git a/serverless/pytorch/foolwood/siammask/nuclio/function.yaml b/serverless/pytorch/foolwood/siammask/nuclio/function.yaml index efdb0ca8..bb165c79 100644 --- a/serverless/pytorch/foolwood/siammask/nuclio/function.yaml +++ b/serverless/pytorch/foolwood/siammask/nuclio/function.yaml @@ -26,21 +26,20 @@ spec: value: /opt/nuclio - kind: RUN value: conda create -y -n siammask python=3.6 - - kind: RUN - value: source activate siammask + - kind: SHELL + value: '["conda", "run", "-n", "siammask", "/bin/bash", "-c"]' - kind: RUN value: git clone https://github.com/foolwood/SiamMask.git - kind: RUN - value: pip install -r SiamMask/requirements.txt + value: pip install -r SiamMask/requirements.txt jsonpickle - kind: RUN value: conda install -y gcc_linux-64 - kind: RUN value: cd SiamMask && bash make.sh && cd - - kind: RUN value: wget -P SiamMask/experiments/siammask_sharp http://www.robots.ox.ac.uk/~qwang/SiamMask_DAVIS.pth - - - kind: WORKDIR - value: /opt/nuclio/pysot + - kind: ENTRYPOINT + value: '["conda", "run", "-n", "siammask"]' triggers: myHttpTrigger: diff --git a/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py b/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py index de79c16d..826c7922 100644 --- a/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py +++ b/serverless/pytorch/foolwood/siammask/nuclio/model_handler.py @@ -4,6 +4,9 @@ from tools.test import * import os +from copy import copy +import jsonpickle +import numpy as np class ModelHandler: def __init__(self): @@ -11,7 +14,8 @@ class ModelHandler: self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.benchmark = True - base_dir = "/opt/nuclio/SiamMask/experiments/siammask_sharp" + base_dir = os.environ.get("MODEL_PATH", + "/opt/nuclio/SiamMask/experiments/siammask_sharp") class configPath: config = os.path.join(base_dir, "config_davis.json") @@ -21,18 +25,42 @@ class ModelHandler: self.siammask = load_pretrain(siammask, os.path.join(base_dir, "SiamMask_DAVIS.pth")) self.siammask.eval().to(self.device) + def encode_state(self, state): + state['net.zf'] = state['net'].zf + state.pop('net', None) + state.pop('mask', None) + + for k,v in state.items(): + state[k] = jsonpickle.encode(v) + + return state + + def decode_state(self, state): + for k,v in state.items(): + state[k] = jsonpickle.decode(v) + + state['net'] = copy(self.siammask) + state['net'].zf = state['net.zf'] + del state['net.zf'] + + return state def infer(self, image, shape, state): + image = np.array(image) if state is None: # init tracking - x, y, w, h = shape - target_pos = np.array([x + w / 2, y + h / 2]) - target_sz = np.array([w, h]) - state = siamese_init(image, target_pos, target_sz, self.siammask, + xtl, ytl, xbr, ybr = shape + target_pos = np.array([(xtl + xbr) / 2, (ytl + ybr) / 2]) + target_sz = np.array([xbr - xtl, ybr - ytl]) + siammask = copy(self.siammask) # don't modify self.siammask + state = siamese_init(image, target_pos, target_sz, siammask, self.config['hp'], device=self.device) + state = self.encode_state(state) else: # track - state = siamese_track(state, image, mask_enable=True, refine_enable=True, - device=self.device) - shape = state['ploygon'].flatten() + state = self.decode_state(state) + state = siamese_track(state, image, mask_enable=True, + refine_enable=True, device=self.device) + shape = state['ploygon'].flatten().tolist() + state = self.encode_state(state) return {"shape": shape, "state": state}