IOG serverless function + some fixes (#2578)
* Initial version of Inside Outside Guidance serverless function * Support neg_points in interactors * Improved deployment process of serverless functions * Improve installation.md for serverless functions. * Update CHANGELOG, use NUCLIO_DASHBOARD_DEFAULT_FUNCTION_MOUNT_MODE as recommended by nuclio developers. * Disable warning from markdown linter about max line length for a table. * Fix IOG function with conda environment * Fix tensorflow matterport/mask_rcnn Co-authored-by: Boris Sekachev <boris.sekachev@intel.com>main
Nikita Manovich
5 years ago
committed by
GitHub
parent
51c3dd8fe3
commit
97cb892844
@ -0,0 +1,71 @@
|
||||
metadata:
|
||||
name: pth.shiyinzhang.iog
|
||||
namespace: cvat
|
||||
annotations:
|
||||
name: IOG
|
||||
type: interactor
|
||||
spec:
|
||||
framework: pytorch
|
||||
min_pos_points: 1
|
||||
startswith_box: true
|
||||
|
||||
spec:
|
||||
description: Interactive Object Segmentation with Inside-Outside Guidance
|
||||
runtime: 'python:3.6'
|
||||
handler: main:handler
|
||||
eventTimeout: 30s
|
||||
env:
|
||||
- name: PYTHONPATH
|
||||
value: /opt/nuclio/iog
|
||||
|
||||
build:
|
||||
image: cvat/pth.shiyinzhang.iog
|
||||
baseImage: continuumio/miniconda3
|
||||
|
||||
directives:
|
||||
preCopy:
|
||||
- kind: WORKDIR
|
||||
value: /opt/nuclio
|
||||
- kind: RUN
|
||||
value: conda create -y -n iog python=3.6
|
||||
- kind: SHELL
|
||||
value: '["conda", "run", "-n", "iog", "/bin/bash", "-c"]'
|
||||
- kind: RUN
|
||||
value: conda install -y -c anaconda curl
|
||||
- kind: RUN
|
||||
value: conda install -y pytorch=0.4 torchvision=0.2 -c pytorch
|
||||
- kind: RUN
|
||||
value: conda install -y -c conda-forge pycocotools opencv scipy
|
||||
- kind: RUN
|
||||
value: git clone https://github.com/shiyinzhang/Inside-Outside-Guidance.git iog
|
||||
- kind: WORKDIR
|
||||
value: /opt/nuclio/iog
|
||||
- kind: ENV
|
||||
value: fileid=1Lm1hhMhhjjnNwO4Pf7SC6tXLayH2iH0l
|
||||
- kind: ENV
|
||||
value: filename=IOG_PASCAL_SBD.pth
|
||||
- kind: RUN
|
||||
value: curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${fileid}"
|
||||
- kind: RUN
|
||||
value: echo "/download/ {print \$NF}" > confirm_code.awk
|
||||
- kind: RUN
|
||||
value: curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk -f confirm_code.awk ./cookie`&id=${fileid}" -o ${filename}
|
||||
- kind: WORKDIR
|
||||
value: /opt/nuclio
|
||||
- kind: ENTRYPOINT
|
||||
value: '["conda", "run", "-n", "iog"]'
|
||||
|
||||
triggers:
|
||||
myHttpTrigger:
|
||||
maxWorkers: 2
|
||||
kind: 'http'
|
||||
workerAvailabilityTimeoutMilliseconds: 10000
|
||||
attributes:
|
||||
maxRequestBodySize: 33554432 # 32MB
|
||||
|
||||
platform:
|
||||
attributes:
|
||||
restartPolicy:
|
||||
name: always
|
||||
maximumRetryCount: 3
|
||||
mountMode: volume
|
||||
@ -0,0 +1,40 @@
|
||||
# Copyright (C) 2020 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import json
|
||||
import base64
|
||||
from PIL import Image
|
||||
import io
|
||||
import numpy as np
|
||||
from model_handler import ModelHandler
|
||||
|
||||
def init_context(context):
|
||||
context.logger.info("Init context... 0%")
|
||||
|
||||
model = ModelHandler()
|
||||
setattr(context.user_data, 'model', model)
|
||||
|
||||
context.logger.info("Init context...100%")
|
||||
|
||||
def handler(context, event):
|
||||
context.logger.info("call handler")
|
||||
data = event.body
|
||||
pos_points = data["pos_points"]
|
||||
neg_points = data["neg_points"]
|
||||
obj_bbox = data.get("obj_bbox", None)
|
||||
threshold = data.get("threshold", 0.8)
|
||||
buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8')))
|
||||
image = Image.open(buf)
|
||||
|
||||
if obj_bbox is None:
|
||||
x, y = np.split(np.transpose(np.array(neg_points)), 2)
|
||||
obj_bbox = [np.min(x), np.min(y), np.max(x), np.max(y)]
|
||||
neg_points = []
|
||||
|
||||
polygon = context.user_data.model.handle(image, obj_bbox,
|
||||
pos_points, neg_points, threshold)
|
||||
return context.Response(body=json.dumps(polygon),
|
||||
headers={},
|
||||
content_type='application/json',
|
||||
status_code=200)
|
||||
@ -0,0 +1,123 @@
|
||||
# Copyright (C) 2020 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import numpy as np
|
||||
import os
|
||||
import cv2
|
||||
import torch
|
||||
from networks.mainnetwork import Network
|
||||
from dataloaders import helpers
|
||||
|
||||
def convert_mask_to_polygon(mask):
|
||||
mask = np.array(mask, dtype=np.uint8)
|
||||
cv2.normalize(mask, mask, 0, 255, cv2.NORM_MINMAX)
|
||||
contours = None
|
||||
if int(cv2.__version__.split('.')[0]) > 3:
|
||||
contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0]
|
||||
else:
|
||||
contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1]
|
||||
|
||||
contours = max(contours, key=lambda arr: arr.size)
|
||||
if contours.shape.count(1):
|
||||
contours = np.squeeze(contours)
|
||||
if contours.size < 3 * 2:
|
||||
raise Exception('Less then three point have been detected. Can not build a polygon.')
|
||||
|
||||
polygon = []
|
||||
for point in contours:
|
||||
polygon.append([int(point[0]), int(point[1])])
|
||||
|
||||
return polygon
|
||||
|
||||
class ModelHandler:
|
||||
def __init__(self):
|
||||
base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio/iog")
|
||||
model_path = os.path.join(base_dir, "IOG_PASCAL_SBD.pth")
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
# Number of input channels (RGB + heatmap of IOG points)
|
||||
self.net = Network(nInputChannels=5, num_classes=1, backbone='resnet101',
|
||||
output_stride=16, sync_bn=None, freeze_bn=False)
|
||||
|
||||
pretrain_dict = torch.load(model_path)
|
||||
self.net.load_state_dict(pretrain_dict)
|
||||
self.net.to(self.device)
|
||||
self.net.eval()
|
||||
|
||||
def handle(self, image, bbox, pos_points, neg_points, threshold):
|
||||
with torch.no_grad():
|
||||
# extract a crop with padding from the image
|
||||
crop_padding = 30
|
||||
crop_bbox = [
|
||||
max(bbox[0] - crop_padding, 0),
|
||||
max(bbox[1] - crop_padding, 0),
|
||||
min(bbox[2] + crop_padding, image.width - 1),
|
||||
min(bbox[3] + crop_padding, image.height - 1)
|
||||
]
|
||||
crop_shape = (
|
||||
int(crop_bbox[2] - crop_bbox[0] + 1), # width
|
||||
int(crop_bbox[3] - crop_bbox[1] + 1), # height
|
||||
)
|
||||
|
||||
# try to use crop_from_bbox(img, bbox, zero_pad) here
|
||||
input_crop = np.array(image.crop(crop_bbox)).astype(np.float32)
|
||||
|
||||
# resize the crop
|
||||
input_crop = cv2.resize(input_crop, (512, 512), interpolation=cv2.INTER_NEAREST)
|
||||
crop_scale = (512 / crop_shape[0], 512 / crop_shape[1])
|
||||
|
||||
def translate_points_to_crop(points):
|
||||
points = [
|
||||
((p[0] - crop_bbox[0]) * crop_scale[0], # x
|
||||
(p[1] - crop_bbox[1]) * crop_scale[1]) # y
|
||||
for p in points]
|
||||
|
||||
return points
|
||||
|
||||
pos_points = translate_points_to_crop(pos_points)
|
||||
neg_points = translate_points_to_crop(neg_points)
|
||||
|
||||
# Create IOG image
|
||||
pos_gt = np.zeros(shape=input_crop.shape[:2], dtype=np.float64)
|
||||
neg_gt = np.zeros(shape=input_crop.shape[:2], dtype=np.float64)
|
||||
for p in pos_points:
|
||||
pos_gt = np.maximum(pos_gt, helpers.make_gaussian(pos_gt.shape, center=p))
|
||||
for p in neg_points:
|
||||
neg_gt = np.maximum(neg_gt, helpers.make_gaussian(neg_gt.shape, center=p))
|
||||
iog_image = np.stack((pos_gt, neg_gt), axis=2).astype(dtype=input_crop.dtype)
|
||||
|
||||
# Convert iog_image to an image (0-255 values)
|
||||
cv2.normalize(iog_image, iog_image, 0, 255, cv2.NORM_MINMAX)
|
||||
|
||||
# Concatenate input crop and IOG image
|
||||
input_blob = np.concatenate((input_crop, iog_image), axis=2)
|
||||
|
||||
# numpy image: H x W x C
|
||||
# torch image: C X H X W
|
||||
input_blob = input_blob.transpose((2, 0, 1))
|
||||
# batch size is 1
|
||||
input_blob = np.array([input_blob])
|
||||
input_tensor = torch.from_numpy(input_blob)
|
||||
|
||||
input_tensor = input_tensor.to(self.device)
|
||||
output_mask = self.net.forward(input_tensor)[4]
|
||||
output_mask = output_mask.to(self.device)
|
||||
pred = np.transpose(output_mask.data.numpy()[0, :, :, :], (1, 2, 0))
|
||||
pred = pred > threshold
|
||||
pred = np.squeeze(pred)
|
||||
|
||||
# Convert a mask to a polygon
|
||||
polygon = convert_mask_to_polygon(pred)
|
||||
def translate_points_to_image(points):
|
||||
points = [
|
||||
(p[0] / crop_scale[0] + crop_bbox[0], # x
|
||||
p[1] / crop_scale[1] + crop_bbox[1]) # y
|
||||
for p in points]
|
||||
|
||||
return points
|
||||
|
||||
polygon = translate_points_to_image(polygon)
|
||||
|
||||
return polygon
|
||||
|
||||
Loading…
Reference in New Issue