Git application (#184)

main
Boris Sekachev 7 years ago committed by Nikita Manovich
parent 1c38c6d3a8
commit 23b6b54137

3
.gitignore vendored

@ -8,6 +8,9 @@
/logs
/components/openvino/*.tgz
/profiles
/ssh/*
!/ssh/README.md
# Ignore temporary files
docker-compose.override.yml

@ -84,6 +84,25 @@
"env": {},
"envFile": "${workspaceFolder}/.env",
},
{
"name": "CVAT git",
"type": "python",
"request": "launch",
"debugStdLib": true,
"stopOnEntry": false,
"pythonPath": "${config:python.pythonPath}",
"program": "${workspaceRoot}/manage.py",
"args": [
"update_git_states"
],
"debugOptions": [
"RedirectOutput",
"DjangoDebugging"
],
"cwd": "${workspaceFolder}",
"env": {},
"envFile": "${workspaceFolder}/.env",
},
],
"compounds": [
{
@ -93,6 +112,7 @@
"CVAT Server",
"CVAT RQ - default",
"CVAT RQ - low",
"CVAT git",
]
}
]

@ -3,11 +3,13 @@ FROM ubuntu:16.04
ARG http_proxy
ARG https_proxy
ARG no_proxy
ARG socks_proxy
ENV TERM=xterm \
http_proxy=${http_proxy} \
https_proxy=${https_proxy} \
no_proxy=${no_proxy}
no_proxy=${no_proxy} \
socks_proxy=${socks_proxy}
ENV LANG='C.UTF-8' \
LC_ALL='C.UTF-8'
@ -102,9 +104,23 @@ COPY cvat/requirements/ /tmp/requirements/
COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
RUN pip3 install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
COPY cvat/ ${HOME}/cvat
COPY ssh ${HOME}/.ssh
# Install git application dependencies
RUN apt-get update && \
apt-get install -y ssh netcat-openbsd git curl zip && \
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
apt-get install -y git-lfs && \
git lfs install && \
rm -rf /var/lib/apt/lists/* && \
if [ -n ${socks_proxy} ]; then \
echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
fi
COPY tests ${HOME}/tests
RUN patch -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch
RUN chown -R ${USER}:${USER} .
RUN chown -R ${USER}:${USER} .
# RUN all commands below as 'django' user
USER ${USER}

@ -25,7 +25,7 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) {
let elem = $(this);
let tid = +elem.attr("id").split("_")[1];
const autoAnnoButton = $("<button> Run auto annotation </button>").addClass("semiBold dashboardButtonUI dashboardAutoAnno");
const autoAnnoButton = $("<button> Run auto annotation </button>").addClass("regular dashboardButtonUI dashboardAutoAnno");
autoAnnoButton.appendTo(elem.find("div.dashboardButtonsUI")[0]);
if (tid in data && data[tid].active) {

@ -5,6 +5,219 @@
*/
"use strict";
/* Server requests */
function createTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) {
$.ajax({
url: "/create/task",
type: "POST",
data: oData,
contentType: false,
processData: false,
success: function(data) {
onSuccessRequest();
requestCreatingStatus(data);
},
error: function(data) {
onComplete();
onError(data.responseText);
}
});
function requestCreatingStatus(data) {
let tid = data.tid;
let request_frequency_ms = 1000;
let done = false;
let requestInterval = setInterval(function() {
$.ajax({
url: "/check/task/" + tid,
success: receiveStatus,
error: function(data) {
clearInterval(requestInterval);
onComplete();
onError(data.responseText);
}
});
}, request_frequency_ms);
function receiveStatus(data) {
if (done) return;
if (data["state"] === "created") {
done = true;
clearInterval(requestInterval);
onComplete();
onSuccessCreate(tid);
}
else if (data["state"] === "error") {
done = true;
clearInterval(requestInterval);
onComplete();
onError(data.stderr);
}
else if (data["state"] === "started" && "status" in data) {
onUpdateStatus(data["status"]);
}
}
}
}
function updateTaskRequest(labels) {
let oData = new FormData();
oData.append("labels", labels);
$.ajax({
url: "/update/task/" + window.cvat.dashboard.taskID,
type: "POST",
data: oData,
contentType: false,
processData: false,
success: function() {
$("#dashboardNewLabels").prop("value", "");
showMessage("Task successfully updated.");
},
error: function(data) {
showMessage("Task update error. " + data.responseText);
},
complete: () => $("#dashboardUpdateModal").addClass("hidden")
});
}
function removeTaskRequest() {
confirm("The action can not be undone. Are you sure?", confirmCallback);
function confirmCallback() {
$.ajax ({
url: "/delete/task/" + window.cvat.dashboard.taskID,
success: function() {
$(`#dashboardTask_${window.cvat.dashboard.taskID}`).remove();
showMessage("Task removed.");
},
error: function(response) {
let message = "Abort. Reason: " + response.responseText;
showMessage(message);
throw Error(message);
}
});
}
}
function uploadAnnotationRequest() {
let input = $("<input>").attr({
type: "file",
accept: "text/xml"
}).on("change", loadXML).click();
function loadXML(e) {
input.remove();
let overlay = showOverlay("File is being uploaded..");
let file = e.target.files[0];
let fileReader = new FileReader();
fileReader.onload = (e) => parseFile(e, overlay);
fileReader.readAsText(file);
}
function parseFile(e, overlay) {
let xmlText = e.target.result;
overlay.setMessage("Request task data from server..");
$.ajax({
url: "/get/task/" + window.cvat.dashboard.taskID,
success: function(data) {
let annotationParser = new AnnotationParser(
{
start: 0,
stop: data.size,
image_meta_data: data.image_meta_data,
flipped: data.flipped
},
new LabelsInfo(data.spec),
new ConstIdGenerator(-1)
);
let asyncParse = function() {
let parsed = null;
try {
parsed = annotationParser.parse(xmlText);
}
catch(error) {
overlay.remove();
showMessage("Parsing errors was occurred. " + error);
return;
}
let asyncSave = function() {
$.ajax({
url: "/delete/annotation/task/" + window.cvat.dashboard.taskID,
type: "DELETE",
success: function() {
asyncSaveChunk(0);
},
error: function(response) {
let message = "Previous annotations cannot be deleted: " +
response.responseText;
showMessage(message);
overlay.remove();
},
});
};
let asyncSaveChunk = function(start) {
const CHUNK_SIZE = 100000;
let end = start + CHUNK_SIZE;
let chunk = {};
let next = false;
for (let prop in parsed) {
if (parsed.hasOwnProperty(prop)) {
chunk[prop] = parsed[prop].slice(start, end);
next |= chunk[prop].length > 0;
}
}
if (next) {
let exportData = createExportContainer();
exportData.create = chunk;
$.ajax({
url: "/save/annotation/task/" + window.cvat.dashboard.taskID,
type: "POST",
data: JSON.stringify(exportData),
contentType: "application/json",
success: function() {
asyncSaveChunk(end);
},
error: function(response) {
let message = "Annotations uploading errors were occurred: " +
response.responseText;
showMessage(message);
overlay.remove();
},
});
} else {
let message = "Annotations were uploaded successfully";
showMessage(message);
overlay.remove();
}
};
overlay.setMessage("Annotation is being saved..");
setTimeout(asyncSave);
};
overlay.setMessage("File is being parsed..");
setTimeout(asyncParse);
},
error: function(response) {
overlay.remove();
let message = "Bad task request: " + response.responseText;
showMessage(message);
throw Error(message);
}
});
}
}
/* Dashboard entrypoint */
window.cvat = window.cvat || {};
@ -15,46 +228,46 @@ window.cvat.config = new Config();
window.cvat.dashboard.uiCallbacks.push(function(elements) {
elements.each(function(idx) {
let elem = $(elements[idx]);
let taskID = +elem.attr('id').split('_')[1];
let taskName = $.trim($( elem.find('label.dashboardTaskNameLabel')[0] ).text());
let buttonsUI = elem.find('div.dashboardButtonsUI')[0];
let taskID = +elem.attr("id").split("_")[1];
let taskName = $.trim($( elem.find("label.dashboardTaskNameLabel")[0] ).text());
let buttonsUI = elem.find("div.dashboardButtonsUI")[0];
let dumpButton = $( $(buttonsUI).find('button.dashboardDumpAnnotation')[0] );
let uploadButton = $( $(buttonsUI).find('button.dashboardUploadAnnotation')[0] );
let updateButton = $( $(buttonsUI).find('button.dashboardUpdateTask')[0] );
let deleteButton = $( $(buttonsUI).find('button.dashboardDeleteTask')[0] );
let dumpButton = $( $(buttonsUI).find("button.dashboardDumpAnnotation")[0] );
let uploadButton = $( $(buttonsUI).find("button.dashboardUploadAnnotation")[0] );
let updateButton = $( $(buttonsUI).find("button.dashboardUpdateTask")[0] );
let deleteButton = $( $(buttonsUI).find("button.dashboardDeleteTask")[0] );
let bugTrackerButton = $(buttonsUI).find('.dashboardOpenTrackerButton');
let bugTrackerButton = $(buttonsUI).find(".dashboardOpenTrackerButton");
if (bugTrackerButton.length) {
bugTrackerButton = $(bugTrackerButton[0]);
bugTrackerButton.on('click', function() {
window.open($(buttonsUI).find('a.dashboardBugTrackerLink').attr('href'));
bugTrackerButton.on("click", function() {
window.open($(buttonsUI).find("a.dashboardBugTrackerLink").attr("href"));
});
}
dumpButton.on('click', function() {
dumpButton.on("click", function() {
window.cvat.dashboard.taskID = taskID;
window.cvat.dashboard.taskName = taskName;
dumpAnnotationRequest(dumpButton, taskID, taskName);
});
uploadButton.on('click', function() {
uploadButton.on("click", function() {
window.cvat.dashboard.taskID = taskID;
window.cvat.dashboard.taskName = taskName;
confirm('The current annotation will be lost. Are you sure?', uploadAnnotationRequest);
confirm("The current annotation will be lost. Are you sure?", uploadAnnotationRequest);
});
updateButton.on('click', function() {
updateButton.on("click", function() {
window.cvat.dashboard.taskID = taskID;
window.cvat.dashboard.taskName = taskName;
$('#dashboardUpdateModal').removeClass('hidden');
$('#dashboardUpdateModal')[0].loadCurrentLabels();
$("#dashboardUpdateModal").removeClass("hidden");
$("#dashboardUpdateModal")[0].loadCurrentLabels();
});
deleteButton.on('click', function() {
deleteButton.on("click", function() {
window.cvat.dashboard.taskID = taskID;
window.cvat.dashboard.taskName = taskName;
RemoveTaskRequest();
removeTaskRequest();
});
});
});
@ -68,54 +281,53 @@ function buildDashboard() {
setupTaskUpdater();
setupSearch();
$(window).on('click', function(e) {
let target = $(e.target);
if ( target.hasClass('modal') ) {
target.addClass('hidden');
$(window).on("click", function(event) {
if (event.target.classList.contains("modal")) {
event.target.classList.add("hidden");
}
});
/* Setup task UIs */
for (let callback of window.cvat.dashboard.uiCallbacks) {
callback( $('.dashboardTaskUI') );
callback( $(".dashboardTaskUI") );
}
$('#loadingOverlay').remove();
$("#loadingOverlay").remove();
}
function setupTaskCreator() {
let dashboardCreateTaskButton = $('#dashboardCreateTaskButton');
let createModal = $('#dashboardCreateModal');
let nameInput = $('#dashboardNameInput');
let labelsInput = $('#dashboardLabelsInput');
let bugTrackerInput = $('#dashboardBugTrackerInput');
let localSourceRadio = $('#dashboardLocalSource');
let shareSourceRadio = $('#dashboardShareSource');
let selectFiles = $('#dashboardSelectFiles');
let filesLabel = $('#dashboardFilesLabel');
let localFileSelector = $('#dashboardLocalFileSelector');
let shareFileSelector = $('#dashboardShareBrowseModal');
let shareBrowseTree = $('#dashboardShareBrowser');
let cancelBrowseServer = $('#dashboardCancelBrowseServer');
let submitBrowseServer = $('#dashboardSubmitBrowseServer');
let flipImagesBox = $('#dashboardFlipImages');
let zOrderBox = $('#dashboardZOrder');
let segmentSizeInput = $('#dashboardSegmentSize');
let customSegmentSize = $('#dashboardCustomSegment');
let overlapSizeInput = $('#dashboardOverlap');
let customOverlapSize = $('#dashboardCustomOverlap');
let imageQualityInput = $('#dashboardImageQuality');
let customCompressQuality = $('#dashboardCustomQuality');
let taskMessage = $('#dashboardCreateTaskMessage');
let submitCreate = $('#dashboardSubmitTask');
let cancelCreate = $('#dashboardCancelTask');
let name = nameInput.prop('value');
let labels = labelsInput.prop('value');
let bugTrackerLink = bugTrackerInput.prop('value');
let source = 'local';
let dashboardCreateTaskButton = $("#dashboardCreateTaskButton");
let createModal = $("#dashboardCreateModal");
let nameInput = $("#dashboardNameInput");
let labelsInput = $("#dashboardLabelsInput");
let bugTrackerInput = $("#dashboardBugTrackerInput");
let localSourceRadio = $("#dashboardLocalSource");
let shareSourceRadio = $("#dashboardShareSource");
let selectFiles = $("#dashboardSelectFiles");
let filesLabel = $("#dashboardFilesLabel");
let localFileSelector = $("#dashboardLocalFileSelector");
let shareFileSelector = $("#dashboardShareBrowseModal");
let shareBrowseTree = $("#dashboardShareBrowser");
let cancelBrowseServer = $("#dashboardCancelBrowseServer");
let submitBrowseServer = $("#dashboardSubmitBrowseServer");
let flipImagesBox = $("#dashboardFlipImages");
let zOrderBox = $("#dashboardZOrder");
let segmentSizeInput = $("#dashboardSegmentSize");
let customSegmentSize = $("#dashboardCustomSegment");
let overlapSizeInput = $("#dashboardOverlap");
let customOverlapSize = $("#dashboardCustomOverlap");
let imageQualityInput = $("#dashboardImageQuality");
let customCompressQuality = $("#dashboardCustomQuality");
let taskMessage = $("#dashboardCreateTaskMessage");
let submitCreate = $("#dashboardSubmitTask");
let cancelCreate = $("#dashboardCancelTask");
let name = nameInput.prop("value");
let labels = labelsInput.prop("value");
let bugTrackerLink = bugTrackerInput.prop("value");
let source = "local";
let flipImages = false;
let zOrder = false;
let segmentSize = 5000;
@ -123,187 +335,191 @@ function setupTaskCreator() {
let compressQuality = 50;
let files = [];
dashboardCreateTaskButton.on('click', function() {
$('#dashboardCreateModal').removeClass('hidden');
dashboardCreateTaskButton.on("click", function() {
$("#dashboardCreateModal").removeClass("hidden");
});
nameInput.on('change', (e) => {name = e.target.value;});
bugTrackerInput.on('change', (e) => {bugTrackerLink = e.target.value;});
labelsInput.on('change', (e) => {labels = e.target.value;});
nameInput.on("change", (e) => {name = e.target.value;});
bugTrackerInput.on("change", (e) => {bugTrackerLink = e.target.value;});
labelsInput.on("change", (e) => {labels = e.target.value;});
localSourceRadio.on('click', function() {
if (source == 'local') return;
source = 'local';
localSourceRadio.on("click", function() {
if (source === "local") {
return;
}
source = "local";
files = [];
updateSelectedFiles();
});
shareSourceRadio.on('click', function() {
if (source == 'share') return;
source = 'share';
shareSourceRadio.on("click", function() {
if (source === "share") {
return;
}
source = "share";
files = [];
updateSelectedFiles();
});
selectFiles.on('click', function() {
if (source == 'local') {
selectFiles.on("click", function() {
if (source === "local") {
localFileSelector.click();
}
else {
shareBrowseTree.jstree("refresh");
shareFileSelector.removeClass('hidden');
shareFileSelector.removeClass("hidden");
shareBrowseTree.jstree({
core: {
data: {
url: 'get_share_nodes',
data: (node) => { return {'id' : node.id}; }
url: "get_share_nodes",
data: (node) => { return {"id" : node.id}; }
}
},
plugins: ['checkbox', 'sort'],
plugins: ["checkbox", "sort"],
});
}
});
localFileSelector.on('change', function(e) {
localFileSelector.on("change", function(e) {
files = e.target.files;
updateSelectedFiles();
});
cancelBrowseServer.on('click', () => shareFileSelector.addClass('hidden'));
submitBrowseServer.on('click', function() {
cancelBrowseServer.on("click", () => shareFileSelector.addClass("hidden"));
submitBrowseServer.on("click", function() {
files = shareBrowseTree.jstree(true).get_selected();
cancelBrowseServer.click();
updateSelectedFiles();
});
flipImagesBox.on('click', (e) => {
flipImagesBox.on("click", (e) => {
flipImages = e.target.checked;
});
zOrderBox.on('click', (e) => {
zOrderBox.on("click", (e) => {
zOrder = e.target.checked;
});
customSegmentSize.on('change', (e) => segmentSizeInput.prop('disabled', !e.target.checked));
customOverlapSize.on('change', (e) => overlapSizeInput.prop('disabled', !e.target.checked));
customCompressQuality.on('change', (e) => imageQualityInput.prop('disabled', !e.target.checked));
customSegmentSize.on("change", (e) => segmentSizeInput.prop("disabled", !e.target.checked));
customOverlapSize.on("change", (e) => overlapSizeInput.prop("disabled", !e.target.checked));
customCompressQuality.on("change", (e) => imageQualityInput.prop("disabled", !e.target.checked));
segmentSizeInput.on('change', function() {
segmentSizeInput.on("change", function() {
let value = Math.clamp(
+segmentSizeInput.prop('value'),
+segmentSizeInput.prop('min'),
+segmentSizeInput.prop('max')
+segmentSizeInput.prop("value"),
+segmentSizeInput.prop("min"),
+segmentSizeInput.prop("max")
);
segmentSizeInput.prop('value', value);
segmentSizeInput.prop("value", value);
segmentSize = value;
});
overlapSizeInput.on('change', function() {
overlapSizeInput.on("change", function() {
let value = Math.clamp(
+overlapSizeInput.prop('value'),
+overlapSizeInput.prop('min'),
+overlapSizeInput.prop('max')
+overlapSizeInput.prop("value"),
+overlapSizeInput.prop("min"),
+overlapSizeInput.prop("max")
);
overlapSizeInput.prop('value', value);
overlapSizeInput.prop("value", value);
overlapSize = value;
});
imageQualityInput.on('change', function() {
imageQualityInput.on("change", function() {
let value = Math.clamp(
+imageQualityInput.prop('value'),
+imageQualityInput.prop('min'),
+imageQualityInput.prop('max')
+imageQualityInput.prop("value"),
+imageQualityInput.prop("min"),
+imageQualityInput.prop("max")
);
imageQualityInput.prop('value', value);
imageQualityInput.prop("value", value);
compressQuality = value;
});
submitCreate.on('click', function() {
submitCreate.on("click", function() {
if (!validateName(name)) {
taskMessage.css('color', 'red');
taskMessage.text('Invalid task name');
taskMessage.css("color", "red");
taskMessage.text("Invalid task name");
return;
}
if (!validateLabels(labels)) {
taskMessage.css('color', 'red');
taskMessage.text('Invalid task labels');
taskMessage.css("color", "red");
taskMessage.text("Invalid task labels");
return;
}
if (!validateSegmentSize(segmentSize)) {
taskMessage.css('color', 'red');
taskMessage.text('Segment size out of range');
taskMessage.css("color", "red");
taskMessage.text("Segment size out of range");
return;
}
if (!validateOverlapSize(overlapSize, segmentSize)) {
taskMessage.css('color', 'red');
taskMessage.text('Overlap size must be positive and not more then segment size');
taskMessage.css("color", "red");
taskMessage.text("Overlap size must be positive and not more then segment size");
return;
}
if (files.length <= 0) {
taskMessage.css('color', 'red');
taskMessage.text('Need specify files for task');
taskMessage.css("color", "red");
taskMessage.text("Need specify files for task");
return;
}
else if (files.length > maxUploadCount && source == 'local') {
taskMessage.css('color', 'red');
taskMessage.text('Too many files. Please use share functionality');
else if (files.length > window.maxUploadCount && source === "local") {
taskMessage.css("color", "red");
taskMessage.text("Too many files. Please use share functionality");
return;
}
else if (source == 'local') {
else if (source === "local") {
let commonSize = 0;
for (let file of files) {
commonSize += file.size;
}
if (commonSize > maxUploadSize) {
taskMessage.css('color', 'red');
taskMessage.text('Too big size. Please use share functionality');
if (commonSize > window.maxUploadSize) {
taskMessage.css("color", "red");
taskMessage.text("Too big size. Please use share functionality");
return;
}
}
let taskData = new FormData();
taskData.append('task_name', name);
taskData.append('bug_tracker_link', bugTrackerLink);
taskData.append('labels', labels);
taskData.append('flip_flag', flipImages);
taskData.append('z_order', zOrder);
taskData.append('storage', source);
if (customSegmentSize.prop('checked')) {
taskData.append('segment_size', segmentSize);
taskData.append("task_name", name);
taskData.append("bug_tracker_link", bugTrackerLink);
taskData.append("labels", labels);
taskData.append("flip_flag", flipImages);
taskData.append("z_order", zOrder);
taskData.append("storage", source);
if (customSegmentSize.prop("checked")) {
taskData.append("segment_size", segmentSize);
}
if (customOverlapSize.prop('checked')) {
taskData.append('overlap_size', overlapSize);
if (customOverlapSize.prop("checked")) {
taskData.append("overlap_size", overlapSize);
}
if (customCompressQuality.prop('checked')) {
taskData.append('compress_quality', compressQuality);
if (customCompressQuality.prop("checked")) {
taskData.append("compress_quality", compressQuality);
}
for (let file of files) {
taskData.append('data', file);
taskData.append("data", file);
}
submitCreate.prop('disabled', true);
submitCreate.prop("disabled", true);
createTaskRequest(taskData,
() => {
taskMessage.css('color', 'green');
taskMessage.text('Successful request! Creating..');
taskMessage.css("color", "green");
taskMessage.text("Successful request! Creating..");
},
() => window.location.reload(),
(response) => {
taskMessage.css('color', 'red');
taskMessage.css("color", "red");
taskMessage.text(response);
},
() => submitCreate.prop('disabled', false),
() => submitCreate.prop("disabled", false),
(status) => {
taskMessage.css('color', 'blue');
taskMessage.css("color", "blue");
taskMessage.text(status);
});
});
@ -311,24 +527,24 @@ function setupTaskCreator() {
function updateSelectedFiles() {
switch (files.length) {
case 0:
filesLabel.text('No Files');
filesLabel.text("No Files");
break;
case 1:
filesLabel.text(typeof(files[0]) == 'string' ? files[0] : files[0].name);
filesLabel.text(typeof(files[0]) === "string" ? files[0] : files[0].name);
break;
default:
filesLabel.text(files.length + ' files');
filesLabel.text(files.length + " files");
}
}
function validateName(name) {
let math = name.match('[a-zA-Z0-9()_ ]+');
let math = name.match("[a-zA-Z0-9()_ ]+");
return math != null;
}
function validateLabels(labels) {
let tmp = labels.replace(/\s/g,'');
let tmp = labels.replace(/\s/g,"");
return tmp.length > 0;
// to do good validator
}
@ -341,278 +557,64 @@ function setupTaskCreator() {
return (overlapSize >= 0 && overlapSize <= segmentSize - 1);
}
cancelCreate.on('click', () => createModal.addClass('hidden'));
cancelCreate.on("click", () => createModal.addClass("hidden"));
}
function setupTaskUpdater() {
let updateModal = $('#dashboardUpdateModal');
let oldLabels = $('#dashboardOldLabels');
let newLabels = $('#dashboardNewLabels');
let submitUpdate = $('#dashboardSubmitUpdate');
let cancelUpdate = $('#dashboardCancelUpdate');
let updateModal = $("#dashboardUpdateModal");
let oldLabels = $("#dashboardOldLabels");
let newLabels = $("#dashboardNewLabels");
let submitUpdate = $("#dashboardSubmitUpdate");
let cancelUpdate = $("#dashboardCancelUpdate");
updateModal[0].loadCurrentLabels = function() {
$.ajax({
url: '/get/task/' + window.cvat.dashboard.taskID,
url: "/get/task/" + window.cvat.dashboard.taskID,
success: function(data) {
let labels = new LabelsInfo(data.spec);
oldLabels.attr('value', labels.normalize());
oldLabels.attr("value", labels.normalize());
},
error: function(response) {
oldLabels.attr('value', 'Bad request');
let message = 'Bad task request: ' + response.responseText;
oldLabels.attr("value", "Bad request");
let message = "Bad task request: " + response.responseText;
throw Error(message);
}
});
};
cancelUpdate.on('click', function() {
$('#dashboardNewLabels').prop('value', '');
updateModal.addClass('hidden');
cancelUpdate.on("click", function() {
$("#dashboardNewLabels").prop("value", "");
updateModal.addClass("hidden");
});
submitUpdate.on('click', () => UpdateTaskRequest(newLabels.prop('value')));
submitUpdate.on("click", () => updateTaskRequest(newLabels.prop("value")));
}
function setupSearch() {
function getUrlParameter(name) {
let regex = new RegExp("[\\?&]" + name + "=([^&#]*)");
let results = regex.exec(window.location.search);
return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
}
let searchInput = $("#dashboardSearchInput");
let searchSubmit = $("#dashboardSearchSubmit");
let line = getUrlParameter('search') || "";
let line = getUrlParameter("search") || "";
searchInput.val(line);
searchSubmit.on('click', function() {
let e = $.Event('keypress');
searchSubmit.on("click", function() {
let e = $.Event("keypress");
e.keyCode = 13;
searchInput.trigger(e);
});
searchInput.on('keypress', function(e) {
searchInput.on("keypress", function(e) {
if (e.keyCode != 13) return;
let filter = e.target.value;
if (!filter) window.location.search = "";
else window.location.search = `search=${filter}`;
});
function getUrlParameter(name) {
let regex = new RegExp('[\\?&]' + name + '=([^&#]*)');
let results = regex.exec(window.location.search);
return results === null ? '' : decodeURIComponent(results[1].replace(/\+/g, ' '));
}
}
/* Server requests */
function createTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) {
$.ajax({
url: '/create/task',
type: 'POST',
data: oData,
contentType: false,
processData: false,
success: function(data) {
onSuccessRequest();
requestCreatingStatus(data);
},
error: function(data) {
onComplete();
onError(data.responseText);
}
});
function requestCreatingStatus(data) {
let tid = data.tid;
let request_frequency_ms = 1000;
let done = false;
let requestInterval = setInterval(function() {
$.ajax({
url: '/check/task/' + tid,
success: receiveStatus,
error: function(data) {
clearInterval(requestInterval);
onComplete();
onError(data.responseText);
}
});
}, request_frequency_ms);
function receiveStatus(data) {
if (done) return;
if (data['state'] == 'created') {
done = true;
clearInterval(requestInterval);
onComplete();
onSuccessCreate();
}
else if (data['state'] == 'error') {
done = true;
clearInterval(requestInterval);
onComplete();
onError(data.stderr);
}
else if (data['state'] == 'started' && 'status' in data) {
onUpdateStatus(data['status']);
}
}
}
}
function UpdateTaskRequest(labels) {
let oData = new FormData();
oData.append('labels', labels);
$.ajax({
url: '/update/task/' + window.cvat.dashboard.taskID,
type: 'POST',
data: oData,
contentType: false,
processData: false,
success: function() {
$('#dashboardNewLabels').prop('value', '');
showMessage('Task successfully updated.');
},
error: function(data) {
showMessage('Task update error. ' + data.responseText);
},
complete: () => $('#dashboardUpdateModal').addClass('hidden')
});
}
function RemoveTaskRequest() {
confirm('The action can not be undone. Are you sure?', confirmCallback);
function confirmCallback() {
$.ajax ({
url: '/delete/task/' + window.cvat.dashboard.taskID,
success: function() {
$(`#dashboardTask_${window.cvat.dashboard.taskID}`).remove();
showMessage('Task removed.');
},
error: function(response) {
let message = 'Abort. Reason: ' + response.responseText;
showMessage(message);
throw Error(message);
}
});
}
}
function uploadAnnotationRequest() {
let input = $('<input>').attr({
type: 'file',
accept: 'text/xml'
}).on('change', loadXML).click();
function loadXML(e) {
input.remove();
let overlay = showOverlay("File is being uploaded..");
let file = e.target.files[0];
let fileReader = new FileReader();
fileReader.onload = (e) => parseFile(e, overlay);
fileReader.readAsText(file);
}
function parseFile(e, overlay) {
let xmlText = e.target.result;
overlay.setMessage('Request task data from server..');
$.ajax({
url: '/get/task/' + window.cvat.dashboard.taskID,
success: function(data) {
let annotationParser = new AnnotationParser(
{
start: 0,
stop: data.size,
image_meta_data: data.image_meta_data,
flipped: data.flipped
},
new LabelsInfo(data.spec),
new ConstIdGenerator(-1)
);
let asyncParse = function() {
let parsed = null;
try {
parsed = annotationParser.parse(xmlText);
}
catch(error) {
overlay.remove();
showMessage("Parsing errors was occurred. " + error);
return;
}
let asyncSave = function() {
$.ajax({
url: '/delete/annotation/task/' + window.cvat.dashboard.taskID,
type: 'DELETE',
success: function() {
asyncSaveChunk(0);
},
error: function(response) {
let message = 'Previous annotations cannot be deleted: ' +
response.responseText;
showMessage(message);
overlay.remove();
},
});
};
let asyncSaveChunk = function(start) {
const CHUNK_SIZE = 100000;
let end = start + CHUNK_SIZE;
let chunk = {};
let next = false;
for (let prop in parsed) {
if (parsed.hasOwnProperty(prop)) {
chunk[prop] = parsed[prop].slice(start, end);
next |= chunk[prop].length > 0;
}
}
if (next) {
let exportData = createExportContainer();
exportData.create = chunk;
$.ajax({
url: '/save/annotation/task/' + window.cvat.dashboard.taskID,
type: 'POST',
data: JSON.stringify(exportData),
contentType: 'application/json',
success: function() {
asyncSaveChunk(end);
},
error: function(response) {
let message = 'Annotations uploading errors were occurred: ' +
response.responseText;
showMessage(message);
overlay.remove();
},
});
} else {
let message = 'Annotations were uploaded successfully';
showMessage(message);
overlay.remove();
}
};
overlay.setMessage('Annotation is being saved..');
setTimeout(asyncSave);
};
overlay.setMessage('File is being parsed..');
setTimeout(asyncParse);
},
error: function(response) {
overlay.remove();
let message = 'Bad task request: ' + response.responseText;
showMessage(message);
throw Error(message);
}
});
}
}

@ -14,7 +14,7 @@
}
.dashboardTaskIntro {
width: 25%;
width: 30%;
height: 75%;
float: left;
margin-left: 20px;
@ -25,7 +25,7 @@
.dashboardButtonsUI {
margin-top: 1%;
width: 33%;
width: 35%;
height: 75%;
float: left;
overflow-y: auto;
@ -33,19 +33,19 @@
.dashboardButtonUI {
display: block;
width: 70%;
height: 2.5em;
width: 60%;
height: 1.6em;
margin: auto;
margin-top: 0.1em;
margin-top: 0.3em;
font-size: 1em;
}
.dashboardJobsUI {
width: 40%;
width: 30%;
height: 75%;
float: left;
text-align: center;
overflow-y: scroll;
overflow-y: auto;
}
.dashboardJobList {

@ -164,13 +164,13 @@ Example: @select=race:__undefined__,skip,asian,black,caucasian,other'/>
</div>
</div>
<div style="width: 100%; height: 14%; padding-top: 10px;">
<div style="float: left; height: 50px; overflow: auto; width: 63%; height: auto;">
<label id="dashboardCreateTaskMessage" class="regular h2 selectable" style="float:left;"> </label>
</div>
<div style="float: right; width: 35%; height: 50px;">
<button id="dashboardCancelTask" class="regular h2"> Cancel </button>
<button id="dashboardSubmitTask" class="regular h2"> Submit </button>
</div>
<div style="float: left; height: 50px; overflow: auto; width: 100%; height: auto; word-break: break-word;">
<label id="dashboardCreateTaskMessage" class="regular h2 selectable" style="float:left;"> </label>
</div>
</div>
</form>
</div>

@ -12,12 +12,12 @@
</center>
<div class="dashboardTaskIntro" style='background-image: url("/get/task/{{item.id}}/frame/0")'> </div>
<div class="dashboardButtonsUI">
<button class="dashboardDumpAnnotation semiBold dashboardButtonUI"> Dump Annotation </button>
<button class="dashboardUploadAnnotation semiBold dashboardButtonUI"> Upload Annotation </button>
<button class="dashboardUpdateTask semiBold dashboardButtonUI"> Update Task </button>
<button class="dashboardDeleteTask semiBold dashboardButtonUI"> Delete Task </button>
<button class="dashboardDumpAnnotation regular dashboardButtonUI"> Dump Annotation </button>
<button class="dashboardUploadAnnotation regular dashboardButtonUI"> Upload Annotation </button>
<button class="dashboardUpdateTask regular dashboardButtonUI"> Update Task </button>
<button class="dashboardDeleteTask regular dashboardButtonUI"> Delete Task </button>
{%if item.bug_tracker %}
<button class="dashboardOpenTrackerButton semiBold dashboardButtonUI"> Open Bug Tracker </button>
<button class="dashboardOpenTrackerButton regular dashboardButtonUI"> Open Bug Tracker </button>
<a class="dashboardBugTrackerLink" href='{{item.bug_tracker}}' style="display: none;"> </a>
{% endif %}
</div>

@ -3,7 +3,5 @@
#
# SPDX-License-Identifier: MIT
from django.test import TestCase
# Create your tests here.

@ -1,5 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

@ -20,6 +20,7 @@ from django.conf import settings
from django.db import transaction
from cvat.apps.profiler import silk_profile
from cvat.apps.engine.plugins import plugin_decorator
from . import models
from .task import get_frame_path, get_image_meta_cache
from .log import slogger
@ -34,7 +35,7 @@ def dump(tid, data_format, scheme, host):
Dump annotation for the task in specified data format.
"""
queue = django_rq.get_queue('default')
queue.enqueue_call(func=_dump, args=(tid, data_format, scheme, host),
queue.enqueue_call(func=_dump, args=(tid, data_format, scheme, host, OrderedDict()),
job_id="annotation.dump/{}".format(tid))
def check(tid):
@ -72,6 +73,7 @@ def get(jid):
return annotation.to_client()
@silk_profile(name="Save job")
@plugin_decorator
@transaction.atomic
def save_job(jid, data):
"""
@ -89,8 +91,13 @@ def save_job(jid, data):
annotation.save_to_db(data['create'])
annotation.update_in_db(data['update'])
db_job.segment.task.updated_date = timezone.now()
db_job.segment.task.save()
updated = sum([ len(data["update"][key]) for key in data["update"] ])
deleted = sum([ len(data["delete"][key]) for key in data["delete"] ])
created = sum([ len(data["create"][key]) for key in data["create"] ])
if updated or deleted or created:
db_job.segment.task.updated_date = timezone.now()
db_job.segment.task.save()
db_job.max_shape_id = max(db_job.max_shape_id, max(client_ids['create']) if client_ids['create'] else -1)
db_job.save()
@ -1497,12 +1504,13 @@ class _AnnotationForSegment(_Annotation):
self.points = annotation.points
self.points_paths = annotation.points_paths
@plugin_decorator
@transaction.atomic
def _dump(tid, data_format, scheme, host):
def _dump(tid, data_format, scheme, host, plugin_meta_data):
db_task = models.Task.objects.select_for_update().get(id=tid)
annotation = _AnnotationForTask(db_task)
annotation.init_from_db()
annotation.dump(data_format, scheme, host)
annotation.dump(data_format, scheme, host, plugin_meta_data)
def _calc_box_area(box):
return (box.xbr - box.xtl) * (box.ybr - box.ytl)
@ -1881,7 +1889,7 @@ class _AnnotationForTask(_Annotation):
# We don't have old boxes on the frame. Let's add all new ones.
self.boxes.extend(int_boxes_by_frame[frame])
def dump(self, data_format, scheme, host):
def dump(self, data_format, scheme, host, plugin_meta_data):
def _flip_box(box, im_w, im_h):
box.xbr, box.xtl = im_w - box.xtl, im_w - box.xbr
box.ybr, box.ytl = im_h - box.ytl, im_h - box.ybr
@ -1945,6 +1953,8 @@ class _AnnotationForTask(_Annotation):
("dumped", str(timezone.localtime(timezone.now())))
])
meta.update(plugin_meta_data)
if db_task.mode == "interpolation":
meta["task"]["original_size"] = OrderedDict([
("width", str(im_meta_data["original_size"][0]["width"])),

@ -64,7 +64,7 @@ class Task(models.Model):
def get_dump_path(self):
name = re.sub(r'[\\/*?:"<>|]', '_', self.name)
return os.path.join(self.path, "{}.dump".format(name))
return os.path.join(self.path, "{}.xml".format(name))
def get_log_path(self):
return os.path.join(self.path, "task.log")

@ -0,0 +1,70 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from functools import update_wrapper
__plugins = {}
def add_plugin(name, function, order, exc_ok = False):
if order not in ["before", "after"]:
raise Exception("Order may be 'before' or 'after' only. Got {}.".format(order))
if not callable(function):
raise Exception("'function' argument should be a callable element")
if not isinstance(name, str):
raise Exception("'name' argument should be a string. Got {}.".format(type(name)))
if name not in __plugins:
__plugins[name] = {
"before": [],
"after": []
}
if function in __plugins[name][order]:
raise Exception("plugin has been attached already")
__plugins[name][order].append(function)
function.exc_ok = exc_ok
def remove_plugin(name, function):
if name in __plugins:
if function in __plugins[name]["before"]:
__plugins[name]["before"].remove(function)
del function.exc_ok
if function in __plugins[name]["after"]:
__plugins[name]["after"].remove(function)
del function.exc_ok
def plugin_decorator(function_to_decorate):
name = function_to_decorate.__name__
def function_wrapper(*args, **kwargs):
if name in __plugins:
for wrapper in __plugins[name]["before"]:
try:
wrapper(*args, **kwargs)
except Exception as ex:
if not wrapper.exc_ok:
raise ex
result = function_to_decorate(*args, **kwargs)
if name in __plugins:
for wrapper in __plugins[name]["after"]:
try:
wrapper(*args, **kwargs)
except Exception as ex:
if not wrapper.exc_ok:
raise ex
return result
# Copy meta info about wrapped function to wrapper function
update_wrapper(function_wrapper, function_to_decorate)
return function_wrapper

@ -75,7 +75,6 @@ html {
overflow: auto;
background-color: rgb(0,0,0);
background-color: rgba(0,0,0,0.4);
overflow: hidden;
}
.modal-content {

@ -204,7 +204,7 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) {
$(window).on('click', function(event) {
Logger.updateUserActivityTimer();
if (['helpWindow', 'settingsWindow'].indexOf(event.target.id) != -1) {
if (event.target.classList.contains('modal')) {
event.target.classList.add('hidden');
}
});

@ -19,6 +19,7 @@ _MEDIA_MIMETYPES_FILE = os.path.join(_SCRIPT_DIR, "media.mimetypes")
mimetypes.init(files=[_MEDIA_MIMETYPES_FILE])
from cvat.apps.engine.models import StatusChoice
from cvat.apps.engine.plugins import plugin_decorator
import django_rq
from django.conf import settings
@ -637,12 +638,9 @@ def _save_task_to_db(db_task, task_params):
db_task.save()
@plugin_decorator
@transaction.atomic
def _create_thread(tid, params):
def raise_exception(images, dirs, videos, archives):
raise Exception('Only one archive, one video or many images can be dowloaded simultaneously. \
{} image(s), {} dir(s), {} video(s), {} archive(s) found'.format(images, dirs, videos, archives))
slogger.glob.info("create task #{}".format(tid))
job = rq.get_current_job()

@ -73,7 +73,7 @@
<template id="messageTemplate">
<div class="modal">
<div class="modal-content" style="width: 400px; height: auto; max-height: 600px; overflow: auto;">
<label class="regular templateMessage selectable"> </label>
<label class="regular templateMessage selectable" style="word-break: break-word;"> </label>
<center>
<button class="regular h2 templateOKButton" style="margin-top: 20px"> Ok </button>
</center>

@ -104,8 +104,10 @@ def create_task(request):
return JsonResponse({'tid': db_task.id})
@login_required
@permission_required(perm=['engine.task.access'],
fn=objectgetter(models.Task, 'tid'), raise_exception=True)
#@permission_required(perm=['engine.task.access'],
# fn=objectgetter(models.Task, 'tid'), raise_exception=True)
# We have commented lines above because the objectgetter() will raise 404 error in
# cases when a task creating ends with an error. So an user don't get an actual reason of an error.
def check_task(request, tid):
"""Check the status of a task"""
try:

@ -0,0 +1,17 @@
## Git Integration For Annotation Storage
### Description
The application allows to integrate any git repository like an annotation storage for a CVAT task.
It supports github or gitlab repositories.
The SSH protocol is used for an authorization.
### Using
* Put a private SSH key into the ```ssh``` directory. The public key corresponding to this private key should be attached to an github user.
* If you don't put any custom key, it will generated automatically.
* Setup a repository URL and a path (which is relative for a repository) in the create task dialog.
* Annotate a task.
* Press the button "Git Repository Sync" on the dashboard.
* In the dialog window press the button "Sync" and waiting for some time.
* An annotation will be dumped, archived and pushed to the attached remote repository. You can do a pull request manually.

@ -0,0 +1,7 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from cvat.settings.base import JS_3RDPARTY
JS_3RDPARTY['dashboard'] = JS_3RDPARTY.get('dashboard', []) + ['git/js/dashboardPlugin.js']

@ -0,0 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

@ -0,0 +1,9 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.apps import AppConfig
class GitConfig(AppConfig):
name = 'git'

@ -0,0 +1,473 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.db import transaction
from django.utils import timezone
from cvat.apps.engine.log import slogger
from cvat.apps.engine.models import Task, Job, User
from cvat.apps.engine.annotation import _dump as dump, FORMAT_XML
from cvat.apps.engine.plugins import add_plugin
from cvat.apps.git.models import GitStatusChoice
from cvat.apps.git.models import GitData
from collections import OrderedDict
import subprocess
import django_rq
import shutil
import json
import git
import os
import re
import rq
def _have_no_access_exception(ex):
if 'Permission denied' in ex.stderr or 'Could not read from remote repository' in ex.stderr:
keys = subprocess.run(['ssh-add -L'], shell = True,
stdout = subprocess.PIPE).stdout.decode('utf-8').split('\n')
keys = list(filter(len, list(map(lambda x: x.strip(), keys))))
raise Exception(
'Could not connect to the remote repository. ' +
'Please make sure you have the correct access rights and the repository exists. ' +
'Available public keys are: ' + str(keys)
)
else:
raise ex
class Git:
__url = None
__path = None
__tid = None
__task_name = None
__branch_name = None
__user = None
__cwd = None
__rep = None
__diffs_dir = None
__annotation_file = None
__sync_date = None
def __init__(self, db_git, tid, user):
self.__db_git = db_git
self.__url = db_git.url
self.__path = db_git.path
self.__tid = tid
self.__user = {
"name": user.username,
"email": user.email or "dummy@cvat.com"
}
self.__cwd = os.path.join(os.getcwd(), "data", str(tid), "repos")
self.__diffs_dir = os.path.join(os.getcwd(), "data", str(tid), "repos_diffs")
self.__task_name = re.sub(r'[\\/*?:"<>|\s]', '_', Task.objects.get(pk = tid).name)[:100]
self.__branch_name = 'cvat_{}_{}'.format(tid, self.__task_name)
self.__annotation_file = os.path.join(self.__cwd, self.__path)
self.__sync_date = db_git.sync_date
# Method parses an got URL.
# SSH: git@github.com/proj/repos[.git]
# HTTP/HTTPS: [http://]github.com/proj/repos[.git]
def _parse_url(self):
try:
http_pattern = "([https|http]+)*[://]*([a-zA-Z0-9._-]+.[a-zA-Z]+)/([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)"
ssh_pattern = "([a-zA-Z0-9._-]+)@([a-zA-Z0-9._-]+):([a-zA-Z0-9._-]+)/([a-zA-Z0-9._-]+)"
http_match = re.match(http_pattern, self.__url)
ssh_match = re.match(ssh_pattern, self.__url)
user = "git"
host = None
repos = None
if http_match:
host = http_match.group(2)
repos = "{}/{}".format(http_match.group(3), http_match.group(4))
elif ssh_match:
user = ssh_match.group(1)
host = ssh_match.group(2)
repos = "{}/{}".format(ssh_match.group(3), ssh_match.group(4))
else:
raise Exception("Got URL doesn't sutisfy for regular expression")
if not repos.endswith(".git"):
repos += ".git"
return user, host, repos
except Exception as ex:
slogger.glob.exception('URL parsing errors occured', exc_info = True)
raise ex
# Method creates the main branch if repostory doesn't have any branches
def _create_master_branch(self):
if len(self.__rep.heads):
raise Exception("Some heads already exists")
readme_md_name = os.path.join(self.__cwd, "README.md")
with open(readme_md_name, "w"):
pass
self.__rep.index.add([readme_md_name])
self.__rep.index.commit("CVAT Annotation. Initial commit by {} at {}".format(self.__user["name"], timezone.now()))
self.__rep.git.push("origin", "master")
# Method creates task branch for repository from current master
def _to_task_branch(self):
# Remove user branch from local repository if it exists
if self.__branch_name not in list(map(lambda x: x.name, self.__rep.heads)):
self.__rep.create_head(self.__branch_name)
self.__rep.head.reference = self.__rep.heads[self.__branch_name]
# Method setups a config file for current user
def _update_config(self):
slogger.task[self.__tid].info("User config initialization..")
with self.__rep.config_writer() as cw:
if not cw.has_section("user"):
cw.add_section("user")
cw.set("user", "name", self.__user["name"])
cw.set("user", "email", self.__user["email"])
cw.release()
# Method initializes repos. It setup configuration, creates master branch if need and checkouts to task branch
def _configurate(self):
self._update_config()
if not len(self.__rep.heads):
self._create_master_branch()
self._to_task_branch()
os.makedirs(self.__diffs_dir, exist_ok = True)
def _ssh_url(self):
user, host, repos = self._parse_url()
return "{}@{}:{}".format(user, host, repos)
# Method clones a remote repos to the local storage using SSH and initializes it
def _clone(self):
os.makedirs(self.__cwd)
ssh_url = self._ssh_url()
# Cloning
slogger.task[self.__tid].info("Cloning remote repository from {}..".format(ssh_url))
self.__rep = git.Repo.clone_from(ssh_url, self.__cwd)
# Intitialization
self._configurate()
# Method is some wrapper for clone
# It restores state if any errors have occured
# It useful if merge conflicts have occured during pull
def _reclone(self):
if os.path.exists(self.__cwd):
if not os.path.isdir(self.__cwd):
os.remove(self.__cwd)
else:
# Rename current repository dir
tmp_repo = os.path.abspath(os.path.join(self.__cwd, "..", "tmp_repo"))
os.rename(self.__cwd, tmp_repo)
# Try clone repository
try:
self._clone()
shutil.rmtree(tmp_repo, True)
except Exception as ex:
# Restore state if any errors have occured
if os.path.isdir(self.__cwd):
shutil.rmtree(self.__cwd, True)
os.rename(tmp_repo, self.__cwd)
raise ex
else:
self._clone()
# Method checkouts to master branch and pulls it from remote repos
def _pull(self):
self.__rep.head.reference = self.__rep.heads["master"]
try:
self.__rep.git.pull("origin", "master")
if self.__branch_name in list(map(lambda x: x.name, self.__rep.heads)):
self.__rep.head.reference = self.__rep.heads["master"]
self.__rep.delete_head(self.__branch_name, force=True)
self.__rep.head.reset("HEAD", index=True, working_tree=True)
self._to_task_branch()
except git.exc.GitError:
# Merge conflicts
self._reclone()
# Method connects a local repository if it exists
# Otherwise it clones it before
def init_repos(self, wo_remote = False):
try:
# Try to use a local repos. It can throw GitError exception
self.__rep = git.Repo(self.__cwd)
self._configurate()
# Check if remote URL is actual
if self._ssh_url() != self.__rep.git.remote('get-url', '--all', 'origin'):
slogger.task[self.__tid].info("Local repository URL is obsolete.")
# We need reinitialize repository if it's false
raise git.exc.GitError("Actual and saved repository URLs aren't match")
except git.exc.GitError:
if wo_remote:
raise Exception('Local repository is failed')
slogger.task[self.__tid].info("Local repository initialization..")
shutil.rmtree(self.__cwd, True)
self._clone()
# Method prepares an annotation, merges diffs and pushes it to remote repository to user branch
def push(self, scheme, host, format, last_save):
# Helpful function which merges diffs
def _accumulate(source, target, target_key):
if isinstance(source, dict):
if target_key is not None and target_key not in target:
target[target_key] = {}
for key in source:
if target_key is not None:
_accumulate(source[key], target[target_key], key)
else:
_accumulate(source[key], target, key)
elif isinstance(source, int):
if source:
if target_key is not None and target_key not in target:
target[target_key] = 0
target[target_key] += source
else:
raise Exception("Unhandled accumulate type: {}".format(type(source)))
# Update local repository
self._pull()
os.makedirs(os.path.join(self.__cwd, os.path.dirname(self.__annotation_file)), exist_ok = True)
# Remove old annotation file if it exists
if os.path.exists(self.__annotation_file):
os.remove(self.__annotation_file)
# Dump an annotation
dump(self.__tid, format, scheme, host, OrderedDict())
dump_name = Task.objects.get(pk = self.__tid).get_dump_path()
ext = os.path.splitext(self.__path)[1]
if ext == '.zip':
subprocess.call('zip -j -r "{}" "{}"'.format(self.__annotation_file, dump_name), shell=True)
elif ext == '.xml':
shutil.copyfile(dump_name, self.__annotation_file)
else:
raise Exception("Got unknown annotation file type")
# Setup LFS for *.zip files
self.__rep.git.lfs("track", self.__path)
self.__rep.git.add(self.__annotation_file)
# Merge diffs
summary_diff = {}
for diff_name in list(map(lambda x: os.path.join(self.__diffs_dir, x), os.listdir(self.__diffs_dir))):
with open(diff_name, 'r') as f:
diff = json.loads(f.read())
_accumulate(diff, summary_diff, None)
# Commit and push
self.__rep.index.add([
'.gitattributes',
])
self.__rep.index.commit("CVAT Annotation updated by {}. Summary: {}".format(self.__user["name"], str(summary_diff)))
self.__rep.git.push("origin", self.__branch_name, "--force")
shutil.rmtree(self.__diffs_dir, True)
# Method checks status of repository annotation
def remote_status(self, last_save):
# Check repository exists and archive exists
if not os.path.isfile(self.__annotation_file) or last_save != self.__sync_date:
return GitStatusChoice.NON_SYNCED
else:
self.__rep.git.update_ref('-d', 'refs/remotes/origin/{}'.format(self.__branch_name))
self.__rep.git.remote('-v', 'update')
last_hash = self.__rep.git.show_ref('refs/heads/{}'.format(self.__branch_name), '--hash')
merge_base_hash = self.__rep.merge_base('refs/remotes/origin/master', self.__branch_name)[0].hexsha
if last_hash == merge_base_hash:
return GitStatusChoice.MERGED
else:
try:
self.__rep.git.show_ref('refs/remotes/origin/{}'.format(self.__branch_name), '--hash')
return GitStatusChoice.SYNCED
except git.exc.GitCommandError:
# Remote branch has been deleted w/o merge
return GitStatusChoice.NON_SYNCED
def _initial_create(tid, params):
if 'git_path' in params:
try:
job = rq.get_current_job()
job.meta['status'] = 'Cloning a repository..'
job.save_meta()
user = params['owner']
git_path = params['git_path']
db_task = Task.objects.get(pk = tid)
path_pattern = r"\[(.+)\]"
path_search = re.search(path_pattern, git_path)
path = None
if path_search is not None:
path = path_search.group(1)
git_path = git_path[0:git_path.find(path) - 1].strip()
path = os.path.join('/', path.strip())
else:
anno_file = re.sub(r'[\\/*?:"<>|\s]', '_', db_task.name)[:100]
path = '/annotation/{}.zip'.format(anno_file)
path = path[1:]
_split = os.path.splitext(path)
if len(_split) < 2 or _split[1] not in [".xml", ".zip"]:
raise Exception("Only .xml and .zip formats are supported")
db_git = GitData()
db_git.url = git_path
db_git.path = path
db_git.task = db_task
try:
_git = Git(db_git, tid, user)
_git.init_repos()
db_git.save()
except git.exc.GitCommandError as ex:
_have_no_access_exception(ex)
except Exception as ex:
slogger.task[tid].exception('exception occured during git _initial_create', exc_info = True)
raise ex
@transaction.atomic
def push(tid, user, scheme, host):
try:
db_task = Task.objects.get(pk = tid)
db_git = GitData.objects.select_for_update().get(pk = db_task)
try:
_git = Git(db_git, tid, user)
_git.init_repos()
_git.push(scheme, host, FORMAT_XML, db_task.updated_date)
# Update timestamp
db_git.sync_date = db_task.updated_date
db_git.status = GitStatusChoice.SYNCED
db_git.save()
except git.exc.GitCommandError as ex:
_have_no_access_exception(ex)
except Exception as ex:
slogger.task[tid].exception('push to remote repository errors occured', exc_info = True)
raise ex
@transaction.atomic
def get(tid, user):
response = {}
response["url"] = {"value": None}
response["status"] = {"value": None, "error": None}
db_task = Task.objects.get(pk = tid)
if GitData.objects.filter(pk = db_task).exists():
db_git = GitData.objects.select_for_update().get(pk = db_task)
response['url']['value'] = '{} [{}]'.format(db_git.url, db_git.path)
try:
rq_id = "git.push.{}".format(tid)
queue = django_rq.get_queue('default')
rq_job = queue.fetch_job(rq_id)
if rq_job is not None and (rq_job.is_queued or rq_job.is_started):
db_git.status = GitStatusChoice.SYNCING
response['status']['value'] = str(db_git.status)
else:
try:
_git = Git(db_git, tid, user)
_git.init_repos(True)
db_git.status = _git.remote_status(db_task.updated_date)
response['status']['value'] = str(db_git.status)
except git.exc.GitCommandError as ex:
_have_no_access_exception(ex)
except Exception as ex:
db_git.status = GitStatusChoice.NON_SYNCED
response['status']['error'] = str(ex)
db_git.save()
return response
def update_states():
db_git_records = GitData.objects.all()
db_user = User.objects.first()
if db_user is None:
# User hasn't been created yet
return
for db_git in db_git_records:
try:
get(db_git.task_id, db_user)
except Exception:
slogger.glob("Exception occured during a status updating for db_git with tid: {}".format(db_git.task_id))
@transaction.atomic
def _onsave(jid, data):
db_task = Job.objects.select_related('segment__task').get(pk = jid).segment.task
try:
db_git = GitData.objects.select_for_update().get(pk = db_task.id)
diff_dir = os.path.join(os.getcwd(), "data", str(db_task.id), "repos_diffs")
os.makedirs(diff_dir, exist_ok = True)
updated = sum([ len(data["update"][key]) for key in data["update"] ])
deleted = sum([ len(data["delete"][key]) for key in data["delete"] ])
created = sum([ len(data["create"][key]) for key in data["create"] ])
if updated or deleted or created:
diff = {
"update": {key: len(data["update"][key]) for key in data["update"].keys()},
"delete": {key: len(data["delete"][key]) for key in data["delete"].keys()},
"create": {key: len(data["create"][key]) for key in data["create"].keys()}
}
diff_files = list(map(lambda x: os.path.join(diff_dir, x), os.listdir(diff_dir)))
last_num = 0
for f in diff_files:
number = os.path.splitext(os.path.basename(f))[0]
number = int(number) if number.isdigit() else last_num
last_num = max(last_num, number)
with open(os.path.join(diff_dir, "{}.diff".format(last_num + 1)), 'w') as f:
f.write(json.dumps(diff))
db_git.status = GitStatusChoice.NON_SYNCED
db_git.save()
except GitData.DoesNotExist:
pass
def _ondump(tid, data_format, scheme, host, plugin_meta_data):
db_task = Task.objects.get(pk = tid)
try:
db_git = GitData.objects.get(pk = db_task)
plugin_meta_data['git'] = OrderedDict({
"url": db_git.url,
"path": db_git.path,
})
except GitData.DoesNotExist:
pass
add_plugin("save_job", _onsave, "after", exc_ok = False)
add_plugin("_create_thread", _initial_create, "before", exc_ok = False)
add_plugin("_dump", _ondump, "before", exc_ok = False)

@ -0,0 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

@ -0,0 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

@ -0,0 +1,21 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.core.management.base import BaseCommand
from cvat.apps.git.git import update_states
import time
INTERVAL_SEC = 600
class Command(BaseCommand):
help = 'Run a regular updating for git status'
def handle(self, *args, **options):
while True:
try:
update_states()
except Exception as ex:
print("An error occured during update task statuses: {}".format(str(ex)))
time.sleep(INTERVAL_SEC)

@ -0,0 +1,26 @@
# Generated by Django 2.1.3 on 2018-12-05 13:24
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('engine', '0014_job_max_shape_id'),
]
operations = [
migrations.CreateModel(
name='GitData',
fields=[
('task', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='engine.Task')),
('url', models.URLField(max_length=2000)),
('path', models.CharField(max_length=256)),
('sync_date', models.DateTimeField(auto_now_add=True)),
('status', models.CharField(default='!sync', max_length=20)),
],
),
]

@ -0,0 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

@ -0,0 +1,24 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.db import models
from cvat.apps.engine.models import Task
from enum import Enum
class GitStatusChoice(Enum):
NON_SYNCED = '!sync'
SYNCING = 'syncing'
SYNCED = 'sync'
MERGED = 'merged'
def __str__(self):
return self.value
class GitData(models.Model):
task = models.OneToOneField(Task, on_delete = models.CASCADE, primary_key = True)
url = models.URLField(max_length = 2000)
path = models.CharField(max_length=256)
sync_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=20, default=GitStatusChoice.NON_SYNCED)

@ -0,0 +1,242 @@
/*
* Copyright (C) 2018 Intel Corporation
*
* SPDX-License-Identifier: MIT
*/
"use strict";
window.cvat = window.cvat || {};
window.cvat.dashboard = window.cvat.dashboard || {};
window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || [];
window.cvat.dashboard.uiCallbacks.push(function(newElements) {
$.ajax({
type: "GET",
url: "/git/repository/meta/get",
success: (data) => {
newElements.each(function(idx) {
let elem = $(newElements[idx]);
let tid = +elem.attr("id").split("_")[1];
if (tid in data) {
if (["sync", "syncing"].includes(data[tid])) {
elem.css("background", "floralwhite");
}
else if (data[tid] === "merged") {
elem.css("background", "azure");
}
else {
elem.css("background", "mistyrose");
}
$("<button> Git Repository Sync </button>").addClass("regular dashboardButtonUI").on("click", () => {
let gitDialogWindow = $(`#${window.cvat.git.reposWindowId}`);
gitDialogWindow.attr("current_tid", tid);
gitDialogWindow.removeClass("hidden");
window.cvat.git.updateState();
}).appendTo(elem.find("div.dashboardButtonsUI")[0]);
}
});
},
error: (data) => {
let message = `Can not get git repositories meta info. Code: ${data.status}. Message: ${data.responseText || data.statusText}`;
showMessage(message);
throw Error(message);
}
});
});
window.cvat.git = {
reposWindowId: "gitReposWindow",
closeReposWindowButtonId: "closeGitReposButton",
reposURLTextId: "gitReposURLText",
reposSyncButtonId: "gitReposSyncButton",
labelStatusId: "gitReposLabelStatus",
labelMessageId: "gitReposLabelMessage",
createURLInputTextId: "gitCreateURLInputText",
updateState: () => {
let gitWindow = $(`#${window.cvat.git.reposWindowId}`);
let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`);
let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`);
let reposURLText = $(`#${window.cvat.git.reposURLTextId}`);
let syncButton = $(`#${window.cvat.git.reposSyncButtonId}`);
reposURLText.attr("placeholder", "Waiting for server response..");
reposURLText.prop("value", "");
gitLabelMessage.css("color", "#cccc00").text("Waiting for server response..");
gitLabelStatus.css("color", "#cccc00").text("\u25cc");
syncButton.attr("disabled", true);
let tid = gitWindow.attr("current_tid");
$.get(`/git/repository/get/${tid}`).done(function(data) {
if (!data.url.value) {
gitLabelMessage.css("color", "black").text("Repository is not attached");
reposURLText.prop("value", "");
reposURLText.attr("placeholder", "Repository is not attached");
return;
}
reposURLText.attr("placeholder", "");
reposURLText.prop("value", data.url.value);
if (!data.status.value) {
gitLabelStatus.css("color", "red").text("\u26a0");
gitLabelMessage.css("color", "red").text(data.status.error);
syncButton.attr("disabled", false);
return;
}
if (data.status.value === "!sync") {
gitLabelStatus.css("color", "red").text("\u2606");
gitLabelMessage.css("color", "red").text("Repository is not synchronized");
syncButton.attr("disabled", false);
}
else if (data.status.value === "sync") {
gitLabelStatus.css("color", "#cccc00").text("\u2605");
gitLabelMessage.css("color", "black").text("Synchronized (merge required)");
}
else if (data.status.value === "merged") {
gitLabelStatus.css("color", "darkgreen").text("\u2605");
gitLabelMessage.css("color", "darkgreen").text("Synchronized");
}
else if (data.status.value === "syncing") {
gitLabelMessage.css("color", "#cccc00").text("Synchronization..");
gitLabelStatus.css("color", "#cccc00").text("\u25cc");
}
else {
let message = `Got unknown repository status: ${data.status.value}`;
gitLabelStatus.css("color", "red").text("\u26a0");
gitLabelMessage.css("color", "red").text(message);
throw Error(message);
}
}).fail(function(data) {
gitWindow.addClass("hidden");
let message = `Error occured during get an repos status. ` +
`Code: ${data.status}, text: ${data.responseText || data.statusText}`;
showMessage(message);
throw Error(message);
});
},
};
document.addEventListener("DOMContentLoaded", () => {
$(`
<tr>
<td> <label class="regular h2"> Dataset Repository: </label> </td>
<td> <input type="text" id="${window.cvat.git.createURLInputTextId}" class="regular"` +
`style="width: 90%", placeholder="github.com/user/repos [annotation/<dump_file_name>.zip]" ` +
`title = "Field for a repository URL and a relative path inside the repository. Default repository path is 'annotation/<dump_file_name>.zip'. There are .zip or .xml extenstions are supported."/>` +
`</td>
</tr>`
).insertAfter($("#dashboardBugTrackerInput").parent().parent());
// Wrap create task request function
let originalCreateTaskRequest = window.createTaskRequest;
window.createTaskRequest = function(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus) {
let gitPath = $(`#${window.cvat.git.createURLInputTextId}`).prop("value").replace(/\s/g, "");
if (gitPath.length) {
oData.append("git_path", gitPath);
}
originalCreateTaskRequest(oData, onSuccessRequest, onSuccessCreate, onError, onComplete, onUpdateStatus);
};
/* GIT MODAL WINDOW PLUGIN PART */
$(`<div id="${window.cvat.git.reposWindowId}" class="modal hidden">
<div style="width: 700px; height: auto;" class="modal-content">
<div style="width: 100%; height: 60%; overflow-y: auto;">
<table style="width: 100%;">
<tr>
<td style="width: 20%;">
<label class="regular h2"> Repository URL: </label>
</td>
<td style="width: 80%;" colspan="2">
<input class="regular h2" type="text" style="width: 92%;" id="${window.cvat.git.reposURLTextId}" readonly/>
</td>
</td>
<tr>
<td style="width: 20%;">
<label class="regular h2"> Status: </label>
</td>
<td style="width: 60%;">
<div>
<label class="regular h2" id="${window.cvat.git.labelStatusId}"> </label>
<label class="regular h2" id="${window.cvat.git.labelMessageId}" style="word-break: break-word; user-select: text;"> </label>
</div>
</td>
<td style="width: 20%;">
<button style="width: 70%;" id="${window.cvat.git.reposSyncButtonId}" class="regular h2"> Sync </button>
</td>
</tr>
</table>
</div>
<center>
<button id="${window.cvat.git.closeReposWindowButtonId}" class="regular h1" style="margin-top: 15px;"> Close </button>
</center>
</div>
</div>`).appendTo("body");
let gitWindow = $(`#${window.cvat.git.reposWindowId}`);
let closeRepositoryWindowButton = $(`#${window.cvat.git.closeReposWindowButtonId}`);
let repositorySyncButton = $(`#${window.cvat.git.reposSyncButtonId}`);
let gitLabelMessage = $(`#${window.cvat.git.labelMessageId}`);
let gitLabelStatus = $(`#${window.cvat.git.labelStatusId}`);
closeRepositoryWindowButton.on("click", () => {
gitWindow.addClass("hidden");
});
repositorySyncButton.on("click", () => {
function badResponse(message) {
try {
showMessage(message);
throw Error(message);
}
finally {
window.cvat.git.updateState();
}
}
gitLabelMessage.css("color", "#cccc00").text("Synchronization..");
gitLabelStatus.css("color", "#cccc00").text("\u25cc");
repositorySyncButton.attr("disabled", true);
let tid = gitWindow.attr("current_tid");
$.get(`/git/repository/push/${tid}`).done((data) => {
setTimeout(timeoutCallback, 1000);
function timeoutCallback() {
$.get(`/git/repository/check/${data.rq_id}`).done((data) => {
if (["finished", "failed", "unknown"].indexOf(data.status) != -1) {
if (data.status === "failed") {
let message = data.error;
badResponse(message);
}
else if (data.status === "unknown") {
let message = `Request for pushing returned status "${data.status}".`;
badResponse(message);
}
else {
window.cvat.git.updateState();
}
}
else {
setTimeout(timeoutCallback, 1000);
}
}).fail((data) => {
let message = `Error was occured during pushing an repos entry. ` +
`Code: ${data.status}, text: ${data.responseText || data.statusText}`;
badResponse(message);
});
}
}).fail((data) => {
let message = `Error was occured during pushing an repos entry. ` +
`Code: ${data.status}, text: ${data.responseText || data.statusText}`;
badResponse(message);
});
});
});

@ -0,0 +1,7 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.test import TestCase
# Create your tests here.

@ -0,0 +1,15 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.urls import path
from . import views
urlpatterns = [
path('get/<int:tid>', views.get_repository),
path('push/<int:tid>', views.push_repository),
path('check/<str:rq_id>', views.check_process),
path('meta/get', views.get_meta_info),
]

@ -0,0 +1,82 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT
from django.http import HttpResponseBadRequest, JsonResponse
from rules.contrib.views import permission_required, objectgetter
from cvat.apps.authentication.decorators import login_required
from cvat.apps.engine.log import slogger
from cvat.apps.engine import models
from cvat.apps.git.models import GitData
import cvat.apps.git.git as CVATGit
import django_rq
@login_required
def check_process(request, rq_id):
try:
queue = django_rq.get_queue('default')
rq_job = queue.fetch_job(rq_id)
if rq_job is not None:
if rq_job.is_queued or rq_job.is_started:
return JsonResponse({"status": "processing"})
elif rq_job.is_finished:
return JsonResponse({"status": "finished"})
else:
return JsonResponse({"status": "failed", "error": rq_job.exc_info})
else:
return JsonResponse({"status": "unknown"})
except Exception as ex:
slogger.glob.error("error occured during checking repository request with rq id {}".format(rq_id), exc_info=True)
return HttpResponseBadRequest(str(ex))
@login_required
@permission_required(perm=['engine.task.delete'],
fn=objectgetter(models.Task, 'tid'), raise_exception=True)
def push_repository(request, tid):
try:
slogger.task[tid].info("push repository request")
rq_id = "git.push.{}".format(tid)
queue = django_rq.get_queue('default')
queue.enqueue_call(func = CVATGit.push, args = (tid, request.user, request.scheme, request.get_host()), job_id = rq_id)
return JsonResponse({ "rq_id": rq_id })
except Exception as ex:
try:
slogger.task[tid].error("error occured during pushing repository request", exc_info=True)
except Exception:
pass
return HttpResponseBadRequest(str(ex))
@login_required
@permission_required(perm=['engine.task.access'],
fn=objectgetter(models.Task, 'tid'), raise_exception=True)
def get_repository(request, tid):
try:
slogger.task[tid].info("get repository request")
return JsonResponse(CVATGit.get(tid, request.user))
except Exception as ex:
try:
slogger.task[tid].error("error occured during getting repository info request", exc_info=True)
except Exception:
pass
return HttpResponseBadRequest(str(ex))
@login_required
def get_meta_info(request):
try:
db_git_records = GitData.objects.all()
response = {}
for db_git in db_git_records:
response[db_git.task_id] = db_git.status
return JsonResponse(response, safe = False)
except Exception as ex:
slogger.glob.exception("error occured during get meta request", exc_info = True)
return HttpResponseBadRequest(str(ex))

@ -110,7 +110,7 @@ window.cvat.dashboard.uiCallbacks.push(function(newElements) {
let tfAnnotationButton = $('<button> Run TF Annotation </button>');
tfAnnotationButton.on('click', onTFAnnotationClick.bind(tfAnnotationButton));
tfAnnotationButton.addClass('dashboardTFAnnotationButton semiBold dashboardButtonUI');
tfAnnotationButton.addClass('dashboardTFAnnotationButton regular dashboardButtonUI');
tfAnnotationButton.appendTo(buttonsUI);
if ((tid in data) && (data[tid].active)) {

@ -25,3 +25,4 @@ dj-pagination==2.4.0
python-logstash==0.4.6
django-revproxy==0.9.15
rules==2.0
GitPython==2.1.11

@ -16,6 +16,10 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
import os
import sys
import fcntl
import shutil
import subprocess
from pathlib import Path
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@ -34,6 +38,43 @@ except ImportError:
f.write("SECRET_KEY = '{}'\n".format(get_random_string(50, chars)))
from keys.secret_key import SECRET_KEY
def generate_ssh_keys():
keys_dir = '{}/keys'.format(os.getcwd())
ssh_dir = '{}/.ssh'.format(os.getenv('HOME'))
pidfile = os.path.join(ssh_dir, 'ssh.pid')
with open(pidfile, "w") as pid:
fcntl.flock(pid, fcntl.LOCK_EX)
try:
subprocess.run(['ssh-add {}/*'.format(ssh_dir)], shell = True, stderr = subprocess.PIPE)
keys = subprocess.run(['ssh-add -l'], shell = True,
stdout = subprocess.PIPE).stdout.decode('utf-8').split('\n')
if 'has no identities' in keys[0]:
print('SSH keys were not found')
volume_keys = os.listdir(keys_dir)
if not ('id_rsa' in volume_keys and 'id_rsa.pub' in volume_keys):
print('New pair of keys are being generated')
subprocess.run(['ssh-keygen -b 4096 -t rsa -f {}/id_rsa -q -N ""'.format(ssh_dir)], shell = True)
shutil.copyfile('{}/id_rsa'.format(ssh_dir), '{}/id_rsa'.format(keys_dir))
shutil.copymode('{}/id_rsa'.format(ssh_dir), '{}/id_rsa'.format(keys_dir))
shutil.copyfile('{}/id_rsa.pub'.format(ssh_dir), '{}/id_rsa.pub'.format(keys_dir))
shutil.copymode('{}/id_rsa.pub'.format(ssh_dir), '{}/id_rsa.pub'.format(keys_dir))
else:
print('Copying them from keys volume')
shutil.copyfile('{}/id_rsa'.format(keys_dir), '{}/id_rsa'.format(ssh_dir))
shutil.copymode('{}/id_rsa'.format(keys_dir), '{}/id_rsa'.format(ssh_dir))
shutil.copyfile('{}/id_rsa.pub'.format(keys_dir), '{}/id_rsa.pub'.format(ssh_dir))
shutil.copymode('{}/id_rsa.pub'.format(keys_dir), '{}/id_rsa.pub'.format(ssh_dir))
subprocess.run(['ssh-add', '{}/id_rsa'.format(ssh_dir)], shell = True)
finally:
fcntl.flock(pid, fcntl.LOCK_UN)
try:
generate_ssh_keys()
except Exception:
pass
# Application definition
JS_3RDPARTY = {}
@ -48,6 +89,7 @@ INSTALLED_APPS = [
'cvat.apps.dashboard',
'cvat.apps.authentication',
'cvat.apps.documentation',
'cvat.apps.git',
'django_rq',
'compressor',
'cacheops',

@ -37,6 +37,9 @@ urlpatterns = [
if apps.is_installed('cvat.apps.tf_annotation'):
urlpatterns.append(path('tensorflow/annotation/', include('cvat.apps.tf_annotation.urls')))
if apps.is_installed('cvat.apps.git'):
urlpatterns.append(path('git/repository/', include('cvat.apps.git.urls')))
if apps.is_installed('cvat.apps.auto_annotation'):
urlpatterns.append(path('auto_annotation/', include('cvat.apps.auto_annotation.urls')))

@ -44,6 +44,7 @@ services:
http_proxy:
https_proxy:
no_proxy:
socks_proxy:
TF_ANNOTATION: "no"
USER: "django"
DJANGO_CONFIGURATION: "production"

@ -0,0 +1,22 @@
Put your SSH keys and SSH config here and they will be installed to the CVAT container.
If you have any problems with a git repository cloning inside the CVAT:
* Make sure that SSH keys have been added to the CVAT container:
```bash
docker exec -it cvat bash -ic 'ls .ssh'
```
* If you need a proxy for connecting to the Internet, specify the socks_proxy variable before build the container. For example:
```bash
socks_proxy=proxy-example.com:1080 docker-compose build
```
* Try to clone a repository via SSH directly in the container by the command:
```bash
docker exec -it cvat bash -ic 'cd /tmp -r && git clone <ssh_repository_url>'
```
* Finally try to clone it on your local machine and if it's successful, contact with us via [Gitter chat](https://gitter.im/opencv-cvat) or [Github issues](https://github.com/opencv/cvat/issues).

@ -17,15 +17,28 @@ loglevel=debug ; info, debug, warn, trace
pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live
[program:ssh-agent]
command=bash -c "rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-agent.sock"
priority=1
autorestart=true
[program:rqworker_default]
command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic \
"exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
numprocs=2
process_name=rqworker_default_%(process_num)s
[program:rqworker_low]
command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
"exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
numprocs=1
[program:git_status_updater]
command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
"/usr/bin/python3 ~/manage.py update_git_states"
environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
numprocs=1
[program:runserver]
@ -40,3 +53,4 @@ command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \
exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
--limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \
%(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s"
environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"

Loading…
Cancel
Save