You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
484 lines
16 KiB
Python
484 lines
16 KiB
Python
# Copyright (C) 2018-2019 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
from enum import Enum
|
|
import re
|
|
import os
|
|
|
|
from django.db import models
|
|
from django.conf import settings
|
|
|
|
from django.contrib.auth.models import User
|
|
from django.core.files.storage import FileSystemStorage
|
|
|
|
class SafeCharField(models.CharField):
|
|
def get_prep_value(self, value):
|
|
value = super().get_prep_value(value)
|
|
if value:
|
|
return value[:self.max_length]
|
|
return value
|
|
|
|
class StatusChoice(str, Enum):
|
|
ANNOTATION = 'annotation'
|
|
VALIDATION = 'validation'
|
|
COMPLETED = 'completed'
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class DataChoice(str, Enum):
|
|
VIDEO = 'video'
|
|
IMAGESET = 'imageset'
|
|
LIST = 'list'
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class StorageMethodChoice(str, Enum):
|
|
CACHE = 'cache'
|
|
FILE_SYSTEM = 'file_system'
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class StorageChoice(str, Enum):
|
|
#AWS_S3 = 'aws_s3_bucket'
|
|
LOCAL = 'local'
|
|
SHARE = 'share'
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class Data(models.Model):
|
|
chunk_size = models.PositiveIntegerField(null=True)
|
|
size = models.PositiveIntegerField(default=0)
|
|
image_quality = models.PositiveSmallIntegerField(default=50)
|
|
start_frame = models.PositiveIntegerField(default=0)
|
|
stop_frame = models.PositiveIntegerField(default=0)
|
|
frame_filter = models.CharField(max_length=256, default="", blank=True)
|
|
compressed_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(),
|
|
default=DataChoice.IMAGESET)
|
|
original_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(),
|
|
default=DataChoice.IMAGESET)
|
|
storage_method = models.CharField(max_length=15, choices=StorageMethodChoice.choices(), default=StorageMethodChoice.FILE_SYSTEM)
|
|
storage = models.CharField(max_length=15, choices=StorageChoice.choices(), default=StorageChoice.LOCAL)
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
def get_frame_step(self):
|
|
match = re.search("step\s*=\s*([1-9]\d*)", self.frame_filter)
|
|
return int(match.group(1)) if match else 1
|
|
|
|
def get_data_dirname(self):
|
|
return os.path.join(settings.MEDIA_DATA_ROOT, str(self.id))
|
|
|
|
def get_upload_dirname(self):
|
|
return os.path.join(self.get_data_dirname(), "raw")
|
|
|
|
def get_compressed_cache_dirname(self):
|
|
return os.path.join(self.get_data_dirname(), "compressed")
|
|
|
|
def get_original_cache_dirname(self):
|
|
return os.path.join(self.get_data_dirname(), "original")
|
|
|
|
@staticmethod
|
|
def _get_chunk_name(chunk_number, chunk_type):
|
|
if chunk_type == DataChoice.VIDEO:
|
|
ext = 'mp4'
|
|
elif chunk_type == DataChoice.IMAGESET:
|
|
ext = 'zip'
|
|
else:
|
|
ext = 'list'
|
|
|
|
return '{}.{}'.format(chunk_number, ext)
|
|
|
|
def _get_compressed_chunk_name(self, chunk_number):
|
|
return self._get_chunk_name(chunk_number, self.compressed_chunk_type)
|
|
|
|
def _get_original_chunk_name(self, chunk_number):
|
|
return self._get_chunk_name(chunk_number, self.original_chunk_type)
|
|
|
|
def get_original_chunk_path(self, chunk_number):
|
|
return os.path.join(self.get_original_cache_dirname(),
|
|
self._get_original_chunk_name(chunk_number))
|
|
|
|
def get_compressed_chunk_path(self, chunk_number):
|
|
return os.path.join(self.get_compressed_cache_dirname(),
|
|
self._get_compressed_chunk_name(chunk_number))
|
|
|
|
def get_preview_path(self):
|
|
return os.path.join(self.get_data_dirname(), 'preview.jpeg')
|
|
|
|
def get_meta_path(self):
|
|
return os.path.join(self.get_upload_dirname(), 'meta_info.txt')
|
|
|
|
def get_dummy_chunk_path(self, chunk_number):
|
|
return os.path.join(self.get_upload_dirname(), 'dummy_{}.txt'.format(chunk_number))
|
|
|
|
class Video(models.Model):
|
|
data = models.OneToOneField(Data, on_delete=models.CASCADE, related_name="video", null=True)
|
|
path = models.CharField(max_length=1024, default='')
|
|
width = models.PositiveIntegerField()
|
|
height = models.PositiveIntegerField()
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
class Image(models.Model):
|
|
data = models.ForeignKey(Data, on_delete=models.CASCADE, related_name="images", null=True)
|
|
path = models.CharField(max_length=1024, default='')
|
|
frame = models.PositiveIntegerField()
|
|
width = models.PositiveIntegerField()
|
|
height = models.PositiveIntegerField()
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
class Project(models.Model):
|
|
name = SafeCharField(max_length=256)
|
|
owner = models.ForeignKey(User, null=True, blank=True,
|
|
on_delete=models.SET_NULL, related_name="+")
|
|
assignee = models.ForeignKey(User, null=True, blank=True,
|
|
on_delete=models.SET_NULL, related_name="+")
|
|
bug_tracker = models.CharField(max_length=2000, blank=True, default="")
|
|
created_date = models.DateTimeField(auto_now_add=True)
|
|
updated_date = models.DateTimeField(auto_now_add=True)
|
|
status = models.CharField(max_length=32, choices=StatusChoice.choices(),
|
|
default=StatusChoice.ANNOTATION)
|
|
|
|
def get_project_dirname(self):
|
|
return os.path.join(settings.PROJECTS_ROOT, str(self.id))
|
|
|
|
def get_project_logs_dirname(self):
|
|
return os.path.join(self.get_project_dirname(), 'logs')
|
|
|
|
def get_client_log_path(self):
|
|
return os.path.join(self.get_project_logs_dirname(), "client.log")
|
|
|
|
def get_log_path(self):
|
|
return os.path.join(self.get_project_logs_dirname(), "project.log")
|
|
|
|
# Extend default permission model
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
class Task(models.Model):
|
|
project = models.ForeignKey(Project, on_delete=models.CASCADE,
|
|
null=True, blank=True, related_name="tasks",
|
|
related_query_name="task")
|
|
name = SafeCharField(max_length=256)
|
|
mode = models.CharField(max_length=32)
|
|
owner = models.ForeignKey(User, null=True, blank=True,
|
|
on_delete=models.SET_NULL, related_name="owners")
|
|
assignee = models.ForeignKey(User, null=True, blank=True,
|
|
on_delete=models.SET_NULL, related_name="assignees")
|
|
bug_tracker = models.CharField(max_length=2000, blank=True, default="")
|
|
created_date = models.DateTimeField(auto_now_add=True)
|
|
updated_date = models.DateTimeField(auto_now=True)
|
|
overlap = models.PositiveIntegerField(null=True)
|
|
# Zero means that there are no limits (default)
|
|
segment_size = models.PositiveIntegerField(default=0)
|
|
status = models.CharField(max_length=32, choices=StatusChoice.choices(),
|
|
default=StatusChoice.ANNOTATION)
|
|
data = models.ForeignKey(Data, on_delete=models.CASCADE, null=True, related_name="tasks")
|
|
|
|
# Extend default permission model
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
def get_task_dirname(self):
|
|
return os.path.join(settings.TASKS_ROOT, str(self.id))
|
|
|
|
def get_task_logs_dirname(self):
|
|
return os.path.join(self.get_task_dirname(), 'logs')
|
|
|
|
def get_client_log_path(self):
|
|
return os.path.join(self.get_task_logs_dirname(), "client.log")
|
|
|
|
def get_log_path(self):
|
|
return os.path.join(self.get_task_logs_dirname(), "task.log")
|
|
|
|
def get_task_artifacts_dirname(self):
|
|
return os.path.join(self.get_task_dirname(), 'artifacts')
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
# Redefined a couple of operation for FileSystemStorage to avoid renaming
|
|
# or other side effects.
|
|
class MyFileSystemStorage(FileSystemStorage):
|
|
def get_valid_name(self, name):
|
|
return name
|
|
|
|
def get_available_name(self, name, max_length=None):
|
|
if self.exists(name) or (max_length and len(name) > max_length):
|
|
raise IOError('`{}` file already exists or its name is too long'.format(name))
|
|
return name
|
|
|
|
def upload_path_handler(instance, filename):
|
|
return os.path.join(instance.data.get_upload_dirname(), filename)
|
|
|
|
# For client files which the user is uploaded
|
|
class ClientFile(models.Model):
|
|
data = models.ForeignKey(Data, on_delete=models.CASCADE, null=True, related_name='client_files')
|
|
file = models.FileField(upload_to=upload_path_handler,
|
|
max_length=1024, storage=MyFileSystemStorage())
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
unique_together = ("data", "file")
|
|
|
|
# For server files on the mounted share
|
|
class ServerFile(models.Model):
|
|
data = models.ForeignKey(Data, on_delete=models.CASCADE, null=True, related_name='server_files')
|
|
file = models.CharField(max_length=1024)
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
# For URLs
|
|
class RemoteFile(models.Model):
|
|
data = models.ForeignKey(Data, on_delete=models.CASCADE, null=True, related_name='remote_files')
|
|
file = models.CharField(max_length=1024)
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
class Segment(models.Model):
|
|
task = models.ForeignKey(Task, on_delete=models.CASCADE)
|
|
start_frame = models.IntegerField()
|
|
stop_frame = models.IntegerField()
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
class Job(models.Model):
|
|
segment = models.ForeignKey(Segment, on_delete=models.CASCADE)
|
|
assignee = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL)
|
|
reviewer = models.ForeignKey(User, null=True, blank=True, related_name='review_job_set', on_delete=models.SET_NULL)
|
|
status = models.CharField(max_length=32, choices=StatusChoice.choices(),
|
|
default=StatusChoice.ANNOTATION)
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
|
|
class Label(models.Model):
|
|
task = models.ForeignKey(Task, null=True, blank=True, on_delete=models.CASCADE)
|
|
project = models.ForeignKey(Project, null=True, blank=True, on_delete=models.CASCADE)
|
|
name = SafeCharField(max_length=64)
|
|
color = models.CharField(default='', max_length=8)
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
unique_together = ('task', 'name')
|
|
|
|
class AttributeType(str, Enum):
|
|
CHECKBOX = 'checkbox'
|
|
RADIO = 'radio'
|
|
NUMBER = 'number'
|
|
TEXT = 'text'
|
|
SELECT = 'select'
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class AttributeSpec(models.Model):
|
|
label = models.ForeignKey(Label, on_delete=models.CASCADE)
|
|
name = models.CharField(max_length=64)
|
|
mutable = models.BooleanField()
|
|
input_type = models.CharField(max_length=16,
|
|
choices=AttributeType.choices())
|
|
default_value = models.CharField(max_length=128)
|
|
values = models.CharField(max_length=4096)
|
|
|
|
class Meta:
|
|
default_permissions = ()
|
|
unique_together = ('label', 'name')
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
class AttributeVal(models.Model):
|
|
# TODO: add a validator here to be sure that it corresponds to self.label
|
|
id = models.BigAutoField(primary_key=True)
|
|
spec = models.ForeignKey(AttributeSpec, on_delete=models.CASCADE)
|
|
value = SafeCharField(max_length=4096)
|
|
|
|
class Meta:
|
|
abstract = True
|
|
default_permissions = ()
|
|
|
|
class ShapeType(str, Enum):
|
|
RECTANGLE = 'rectangle' # (x0, y0, x1, y1)
|
|
POLYGON = 'polygon' # (x0, y0, ..., xn, yn)
|
|
POLYLINE = 'polyline' # (x0, y0, ..., xn, yn)
|
|
POINTS = 'points' # (x0, y0, ..., xn, yn)
|
|
CUBOID = 'cuboid' # (x0, y0, ..., x7, y7)
|
|
|
|
@classmethod
|
|
def choices(cls):
|
|
return tuple((x.value, x.name) for x in cls)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class SourceType(str, Enum):
|
|
AUTO = 'auto'
|
|
MANUAL = 'manual'
|
|
|
|
@classmethod
|
|
def choices(self):
|
|
return tuple((x.value, x.name) for x in self)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class ReviewStatus(str, Enum):
|
|
ACCEPTED = 'accepted'
|
|
REJECTED = 'rejected'
|
|
REVIEW_FURTHER = 'review_further'
|
|
|
|
@classmethod
|
|
def choices(self):
|
|
return tuple((x.value, x.name) for x in self)
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
class Annotation(models.Model):
|
|
id = models.BigAutoField(primary_key=True)
|
|
job = models.ForeignKey(Job, on_delete=models.CASCADE)
|
|
label = models.ForeignKey(Label, on_delete=models.CASCADE)
|
|
frame = models.PositiveIntegerField()
|
|
group = models.PositiveIntegerField(null=True)
|
|
source = models.CharField(max_length=16, choices=SourceType.choices(),
|
|
default=str(SourceType.MANUAL), null=True)
|
|
|
|
class Meta:
|
|
abstract = True
|
|
default_permissions = ()
|
|
|
|
class Commit(models.Model):
|
|
id = models.BigAutoField(primary_key=True)
|
|
author = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL)
|
|
version = models.PositiveIntegerField(default=0)
|
|
timestamp = models.DateTimeField(auto_now=True)
|
|
message = models.CharField(max_length=4096, default="")
|
|
|
|
class Meta:
|
|
abstract = True
|
|
default_permissions = ()
|
|
|
|
class JobCommit(Commit):
|
|
job = models.ForeignKey(Job, on_delete=models.CASCADE, related_name="commits")
|
|
|
|
class FloatArrayField(models.TextField):
|
|
separator = ","
|
|
|
|
def from_db_value(self, value, expression, connection):
|
|
if not value:
|
|
return value
|
|
return [float(v) for v in value.split(self.separator)]
|
|
|
|
def to_python(self, value):
|
|
if isinstance(value, list):
|
|
return value
|
|
|
|
return self.from_db_value(value, None, None)
|
|
|
|
def get_prep_value(self, value):
|
|
return self.separator.join(map(str, value))
|
|
|
|
class Shape(models.Model):
|
|
type = models.CharField(max_length=16, choices=ShapeType.choices())
|
|
occluded = models.BooleanField(default=False)
|
|
z_order = models.IntegerField(default=0)
|
|
points = FloatArrayField()
|
|
|
|
class Meta:
|
|
abstract = True
|
|
default_permissions = ()
|
|
|
|
class LabeledImage(Annotation):
|
|
pass
|
|
|
|
class LabeledImageAttributeVal(AttributeVal):
|
|
image = models.ForeignKey(LabeledImage, on_delete=models.CASCADE)
|
|
|
|
class LabeledShape(Annotation, Shape):
|
|
pass
|
|
|
|
class LabeledShapeAttributeVal(AttributeVal):
|
|
shape = models.ForeignKey(LabeledShape, on_delete=models.CASCADE)
|
|
|
|
class LabeledTrack(Annotation):
|
|
pass
|
|
|
|
class LabeledTrackAttributeVal(AttributeVal):
|
|
track = models.ForeignKey(LabeledTrack, on_delete=models.CASCADE)
|
|
|
|
class TrackedShape(Shape):
|
|
id = models.BigAutoField(primary_key=True)
|
|
track = models.ForeignKey(LabeledTrack, on_delete=models.CASCADE)
|
|
frame = models.PositiveIntegerField()
|
|
outside = models.BooleanField(default=False)
|
|
|
|
class TrackedShapeAttributeVal(AttributeVal):
|
|
shape = models.ForeignKey(TrackedShape, on_delete=models.CASCADE)
|
|
|
|
class Profile(models.Model):
|
|
user = models.OneToOneField(User, on_delete=models.CASCADE)
|
|
rating = models.FloatField(default=0.0)
|
|
|
|
class Review(models.Model):
|
|
job = models.ForeignKey(Job, on_delete=models.CASCADE)
|
|
reviewer = models.ForeignKey(User, null=True, blank=True, related_name='reviews', on_delete=models.SET_NULL)
|
|
assignee = models.ForeignKey(User, null=True, blank=True, related_name='reviewed', on_delete=models.SET_NULL)
|
|
estimated_quality = models.FloatField()
|
|
status = models.CharField(max_length=16, choices=ReviewStatus.choices())
|
|
|
|
class Issue(models.Model):
|
|
frame = models.PositiveIntegerField()
|
|
position = FloatArrayField()
|
|
job = models.ForeignKey(Job, on_delete=models.CASCADE)
|
|
review = models.ForeignKey(Review, null=True, blank=True, on_delete=models.SET_NULL)
|
|
owner = models.ForeignKey(User, null=True, blank=True, related_name='issues', on_delete=models.SET_NULL)
|
|
resolver = models.ForeignKey(User, null=True, blank=True, related_name='resolved_issues', on_delete=models.SET_NULL)
|
|
created_date = models.DateTimeField(auto_now_add=True)
|
|
resolved_date = models.DateTimeField(null=True, blank=True)
|
|
|
|
class Comment(models.Model):
|
|
issue = models.ForeignKey(Issue, on_delete=models.CASCADE)
|
|
author = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL)
|
|
message = models.TextField(default='')
|
|
created_date = models.DateTimeField(auto_now_add=True)
|
|
updated_date = models.DateTimeField(auto_now=True)
|