From bbfa880d1fcd6bfb9073640ce7bdfdade1e5c7ba Mon Sep 17 00:00:00 2001 From: Maria Khrustaleva Date: Thu, 5 Nov 2020 22:48:59 +0300 Subject: [PATCH] Added ability to correct upload video with a rotation record in the metadata (#2218) * Added ability to correct upload video with a rotation record in the metadata * fix sizes of rotated preview * fix sizes of rotated frame * Added tests for uploaded video with rotation record in metadata * Used OpenCV instead of PIL * Fixed tests * Update CHANGELOG * fix * Moved function Co-authored-by: Nikita Manovich --- CHANGELOG.md | 2 +- cvat/apps/engine/media_extractors.py | 21 ++++++- cvat/apps/engine/prepare.py | 28 ++++++++- cvat/apps/engine/task.py | 1 + .../tests/assets/test_rotated_90_video.mp4 | Bin 0 -> 8808 bytes cvat/apps/engine/tests/test_rest_api.py | 53 +++++++++++++++++- cvat/apps/engine/utils.py | 14 +++++ 7 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 cvat/apps/engine/tests/assets/test_rotated_90_video.mp4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a5cd4ec..4f4fd382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.2.0] - Unreleased ### Added - - Removed Z-Order flag from task creation process - Ability to login into CVAT-UI with token from api/v1/auth/login () - Added layout grids toggling ('ctrl + alt + Enter') @@ -29,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to upload prepared meta information along with a video when creating a task () - Optional chaining plugin for cvat-canvas and cvat-ui () - MOTS png mask format support () +- Ability to correct upload video with a rotation record in the metadata () ### Changed diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 21430838..b58bf98c 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -14,6 +14,7 @@ import av import numpy as np from pyunpack import Archive from PIL import Image, ImageFile +from cvat.apps.engine.utils import rotate_image # fixes: "OSError:broken data stream" when executing line 72 while loading images downloaded from the web # see: https://stackoverflow.com/questions/42462431/oserror-broken-data-stream-when-reading-image-file @@ -228,6 +229,16 @@ class VideoReader(IMediaReader): for image in packet.decode(): frame_num += 1 if self._has_frame(frame_num - 1): + if packet.stream.metadata.get('rotate'): + old_image = image + image = av.VideoFrame().from_ndarray( + rotate_image( + image.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) + image.pts = old_image.pts yield (image, self._source_path[0], image.pts) def __iter__(self): @@ -252,7 +263,15 @@ class VideoReader(IMediaReader): container = self._get_av_container() stream = container.streams.video[0] preview = next(container.decode(stream)) - return self._get_preview(preview.to_image()) + return self._get_preview(preview.to_image() if not stream.metadata.get('rotate') \ + else av.VideoFrame().from_ndarray( + rotate_image( + preview.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ).to_image() + ) def get_image_size(self, i): image = (next(iter(self)))[0] diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 9465b680..9ee54630 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -6,6 +6,7 @@ import av from collections import OrderedDict import hashlib import os +from cvat.apps.engine.utils import rotate_image class WorkWithVideo: def __init__(self, **kwargs): @@ -24,7 +25,6 @@ class WorkWithVideo: video_stream.thread_type = 'AUTO' return video_stream - class AnalyzeVideo(WorkWithVideo): def check_type_first_frame(self): container = self._open_video_container(self.source_path, mode='r') @@ -76,7 +76,17 @@ class PrepareInfo(WorkWithVideo): @property def frame_sizes(self): + container = self._open_video_container(self.source_path, 'r') frame = next(iter(self.key_frames.values())) + if container.streams.video[0].metadata.get('rotate'): + frame = av.VideoFrame().from_ndarray( + rotate_image( + frame.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) + self._close_video_container(container) return (frame.width, frame.height) def check_key_frame(self, container, video_stream, key_frame): @@ -150,6 +160,14 @@ class PrepareInfo(WorkWithVideo): if frame_number < start_chunk_frame_number: continue elif frame_number < end_chunk_frame_number and not ((frame_number - start_chunk_frame_number) % step): + if video_stream.metadata.get('rotate'): + frame = av.VideoFrame().from_ndarray( + rotate_image( + frame.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) yield frame elif (frame_number - start_chunk_frame_number) % step: continue @@ -177,6 +195,14 @@ class UploadedMeta(PrepareInfo): container.seek(offset=next(iter(self.key_frames.values())), stream=video_stream) for packet in container.demux(video_stream): for frame in packet.decode(): + if video_stream.metadata.get('rotate'): + frame = av.VideoFrame().from_ndarray( + rotate_image( + frame.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) self._close_video_container(container) return (frame.width, frame.height) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index fad3654f..e724d242 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -294,6 +294,7 @@ def _create_thread(tid, data): if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE: for media_type, media_files in media.items(): + if not media_files: continue diff --git a/cvat/apps/engine/tests/assets/test_rotated_90_video.mp4 b/cvat/apps/engine/tests/assets/test_rotated_90_video.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31bd0737b602e0b9b865e006194021449da024d4 GIT binary patch literal 8808 zcmb7~UuYaf9LMMGl53#}O|MDw*Y--P2&OSfFR?xd(FVywLC#_bMr1G93k}`Pop+Z* zE7-%@V2Ke6^$&>zX|-uf9zq*q@DEKOguWOmm3XjV5BlPp_|Pg;t>2lu>|XQF?=9@@ z-0o**Z$3LS`2;A-7mt%xIul2(i6C77h!s_qwTv5<V zz)lvx&J@5d6u>U~npW7dIr1{W;o)?}3+Fc1mX-Otl6T8;Uw(NDZA#`p${mYB9gD&o zi$WZW!W)a)+IFGNJ&~?B@&>1E5!wbhZ7d3HEDCKb3T-S3Z9u1DM6n!gnc#=xTKDPl zE!!z7+yhzLnr|CB%6hkMr>JnJWOd`&$pYA!0@#HD*yRFPTb@q~e$b=6?wR7`jTk%T zhe&xe3wbmPc{B@oGz)n&>&VLlr>vssW~SpG^|g*`pLhd==P@3gBFv1Hykq9HEzkujvlUB+eFK1ll-621IW~kVmtS zN3)Pevyex#w!BK!=ha*zhHiLXT(vx1%o7=nmPJ*E$K|T}I}TOd9fzvk&eOxx2*(<# z+)}z`e+T!NowGb0Pcj#PH=2bwnuR%=olxD zXH{@V=BuJRl2zdy$*TB{WF2{OIHBF&-b~FlAw=K1;3e8NW=&XB4JWu<)o_AC)o_AC z;Vo|Lb1}uTwN-1u3zm1f;UW=njQRpS+Db7wcx6!xURe}_R~E(Kl|?all_=Bd9Bp!{ zq;&oY)BNOANocNushD!Aghb_335m+75)zeDB_t}RN=Ve!R)gueTBEpu>)=kCsjnEp zXrnBOHp-%Cqb!Ox%A#naBV@50YjQ?7q?P}~b#1qe2KN}(wXrC)u_&~$D73LCv`KV= zD3-0QR=s~{aP@FwhefM6^%ObqqFKZuG>ce-W)X|fEMgIwb>!ilS2A7i8sg+FTSZzU z^9(^CN3)Pevyex#kVmtSN3*uPI@G*N=}Pf2Cl7C1Gd@xt%|af{LLSXR9?e3YV(${c zh3A|Zi6rp~+c1;Ib!_t%&u>CtbzxQ|Z||dDaNv=HLbT_z>(g$$KL0Ke3!T8B*Agex zH9dJ)E;o7Y^~=MDxqWM{DfhO^kDFMtr8p4Q6LEa23mb8~l2;4P%H^xhcONqKfe?=P zO?_?ITd=!Dyt1gyJ9u0-Vk(7?{m#;b?~%^Xpl*aE87kRv7C}#MxY1c$N(Q6>R5>LceZ-``;@LvOYqvVpGXLip`6uT2ixyofd zk=^fnRwv#-68<<23SgeSLO!HglI>ERMb} ze)M*foIiqH*N%pgeXWNZTLX>FO-){$