URLs as source data for a task (#483)

* Update CHANGELOG.md

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Installation guide
 - Linear interpolation for a single point
 - Video frame filter
+- Remote data source (list of URLs to create an annotation task)
 
 ### Changed
 - Outside and keyframe buttons in the side panel for all interpolation shapes (they were only for boxes before)

cvat/apps/dashboard/static/dashboard/js/dashboard.js

@@ -489,9 +489,11 @@ class DashboardView {
         const labelsInput = $('#dashboardLabelsInput');
         const bugTrackerInput = $('#dashboardBugTrackerInput');
         const localSourceRadio = $('#dashboardLocalSource');
+        const remoteSourceRadio = $('#dashboardRemoteSource');
         const shareSourceRadio = $('#dashboardShareSource');
         const selectFiles = $('#dashboardSelectFiles');
         const filesLabel = $('#dashboardFilesLabel');
+        const remoteFileInput = $('#dashboardRemoteFileInput');
         const localFileSelector = $('#dashboardLocalFileSelector');
         const shareFileSelector = $('#dashboardShareBrowseModal');
         const shareBrowseTree = $('#dashboardShareBrowser');
@@ -587,15 +589,32 @@ class DashboardView {
         localSourceRadio.on('click', () => {
             if (source === 'local') {
                 return;
+            } else if (source === 'remote') {
+                selectFiles.parent().removeClass('hidden');
+                remoteFileInput.parent().addClass('hidden');
             }
             source = 'local';
             files = [];
             updateSelectedFiles();
         });
 
+        remoteSourceRadio.on('click', () => {
+            if (source === 'remote') {
+                return;
+            }
+            source = 'remote';
+            selectFiles.parent().addClass('hidden');
+            remoteFileInput.parent().removeClass('hidden');
+            remoteFileInput.prop('value', '');
+            files = [];
+        });
+
         shareSourceRadio.on('click', () => {
             if (source === 'share') {
                 return;
+            } else if (source === 'remote') {
+                selectFiles.parent().removeClass('hidden');
+                remoteFileInput.parent().addClass('hidden');
             }
             source = 'share';
             files = [];
@@ -639,6 +658,11 @@ class DashboardView {
             updateSelectedFiles();
         });
 
+        remoteFileInput.on('change', function(e) {
+            let text = remoteFileInput.prop('value');
+            files = text.split('\n').map(f => f.trim()).filter(f => f.length > 0);
+        });
+
         cancelBrowseServer.on('click', () => shareFileSelector.addClass('hidden'));
         submitBrowseServer.on('click', () => {
             if (!createModal.hasClass('hidden')) {
@@ -806,8 +830,10 @@ class DashboardView {
                 let task = new window.cvat.classes.Task(description);
                 if (source === 'local') {
                     task.clientFiles = Array.from(files);
-                } else {
+                } else if (source === 'share') {
                     task.serverFiles = Array.from(files);
+                } else if (source === 'remote') {
+                    task.remoteFiles = Array.from(files);
                 }
                 submitCreate.attr('disabled', true);
                 cancelCreate.attr('disabled', true);

cvat/apps/dashboard/templates/dashboard/dashboard.html

@@ -105,6 +105,7 @@ Example: @select=race:__undefined__,skip,asian,black,caucasian,other'/>
                     <td> <label class="regular h2"> Source: </label> </td>
                     <td>
                         <input id="dashboardLocalSource" type="radio" name="sourceType" value="local" checked=true/> <label for="dashboardLocalSource" class="regular h2" for="localSource"> Local </label>
+                        <br> <input id="dashboardRemoteSource" type="radio" name="sourceType" value="remote"/> <label for="dashboardRemoteSource" class="regular h2"> Remote </label>
                         <br> <input id="dashboardShareSource" type="radio" name="sourceType" value="share"/> <label for="dashboardShareSource" class="regular h2" for="shareSource"> Share </label>
                     </td>
                 </tr>
@@ -179,6 +180,10 @@ Example: @select=race:__undefined__,skip,asian,black,caucasian,other'/>
                     <label id="dashboardFilesLabel" class="regular h2" style="margin-left: 10px"> No Files </label>
                     <input id="dashboardLocalFileSelector" type="file" style="display: none" multiple/>
                 </div>
+                <div class="hidden">
+                    <label for="dashboardRemoteFileInput" class="regular h2"> URLs: </label><br>
+                    <textarea id="dashboardRemoteFileInput" rows="5" placeholder="One URL per line" style="width: 100%;"></textarea>
+                </div>
             </div>
             <div style="width: 100%; height: 14%; padding-top: 10px;">
                 <div style="float: right; width: 35%; height: 50px;">

cvat/apps/engine/serializers.py

@@ -96,6 +96,14 @@ class RemoteFileSerializer(serializers.ModelSerializer):
         model = models.RemoteFile
         fields = ('file', )
 
+    # pylint: disable=no-self-use
+    def to_internal_value(self, data):
+        return {'file': data}
+
+    # pylint: disable=no-self-use
+    def to_representation(self, instance):
+        return instance.file
+
 class RqStatusSerializer(serializers.Serializer):
     state = serializers.ChoiceField(choices=[
         "Queued", "Started", "Finished", "Failed"])

cvat/apps/engine/task.py

@@ -13,6 +13,9 @@ import numpy as np
 from PIL import Image
 from traceback import print_exception
 from ast import literal_eval
+from urllib import error as urlerror
+from urllib import parse as urlparse
+from urllib import request as urlrequest
 
 import mimetypes
 _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
@@ -319,7 +322,7 @@ def _validate_data(data):
     counter = {"image": 0, "video": 0, "archive": 0, "directory": 0}
 
     client_video, client_archive = count_files(
-        file_mapping={ f:f for f in data['client_files']},
+        file_mapping={ f:f for f in data['remote_files'] or data['client_files']},
         counter=counter,
     )
 
@@ -343,6 +346,33 @@ def _validate_data(data):
 
     return client_video or server_video, client_archive or server_archive
 
+def _download_data(urls, upload_dir):
+    job = rq.get_current_job()
+    local_files = {}
+    for url in urls:
+        name = os.path.basename(urlrequest.url2pathname(urlparse.urlparse(url).path))
+        if name in local_files:
+            raise Exception("filename collision: {}".format(name))
+        slogger.glob.info("Downloading: {}".format(url))
+        job.meta['status'] = '{} is being downloaded..'.format(url)
+        job.save_meta()
+
+        req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        try:
+            with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp:
+                while True:
+                    block = fp.read(8192)
+                    if not block:
+                        break
+                    tfp.write(block)
+        except urlerror.HTTPError as err:
+            raise Exception("Failed to download " + url + ". " + str(err.code) + ' - ' + err.reason)
+        except urlerror.URLError as err:
+            raise Exception("Invalid URL: " + url + ". " + err.reason)
+
+        local_files[name] = True
+    return list(local_files.keys())
+
 @transaction.atomic
 def _create_thread(tid, data):
     slogger.glob.info("create task #{}".format(tid))
@@ -352,6 +382,8 @@ def _create_thread(tid, data):
         raise NotImplementedError("Adding more data is not implemented")
 
     upload_dir = db_task.get_upload_dirname()
+    if data['remote_files']:
+        data['remote_files'] = _download_data(data['remote_files'], upload_dir)
     video, archive = _validate_data(data)
 
     if data['server_files']:

cvatjs/src/api-implementation.js

@@ -230,7 +230,7 @@
                 const taskFiles = {
                     client_files: this.clientFiles,
                     server_files: this.serverFiles,
-                    remote_files: [], // hasn't been supported yet
+                    remote_files: this.remoteFiles,
                 };
 
                 const task = await serverProxy.tasks.createTask(taskData, taskFiles, onUpdate);

cvatjs/src/session.js

@@ -761,6 +761,34 @@
                         Array.prototype.push.apply(data.files.client_files, clientFiles);
                     },
                 },
+                /**
+                    * List of files from remote host
+                    * @name remoteFiles
+                    * @type {File[]}
+                    * @memberof module:API.cvat.classes.Task
+                    * @instance
+                    * @throws {module:API.cvat.exceptions.ArgumentError}
+                */
+                remoteFiles: {
+                    get: () => [...data.files.remote_files],
+                    set: (remoteFiles) => {
+                        if (!Array.isArray(remoteFiles)) {
+                            throw new window.cvat.exceptions.ArgumentError(
+                                `Value must be an array. But ${typeof (remoteFiles)} has been got.`,
+                            );
+                        }
+
+                        for (const value of remoteFiles) {
+                            if (typeof (value) !== 'string') {
+                                throw new window.cvat.exceptions.ArgumentError(
+                                    `Array values must be a string. But ${typeof (value)} has been got.`,
+                                );
+                            }
+                        }
+
+                        Array.prototype.push.apply(data.files.remote_files, remoteFiles);
+                    },
+                },
             }));
         }