wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

PolyShape interpolation algorithm through stable marriage problem (#546)

Browse Source
main
Boris Sekachev 7 years ago committed by Nikita Manovich
parent bbdf014442
commit d15de7bc29
1 changed files with 443 additions and 55 deletions
Show Stats Download Patch File Download Diff File

498
cvatjs/src/annotations.js
Unescape Escape View File

@ -152,9 +152,9 @@
get(targetFrame) {
return Object.assign(
{}, this.interpolatePosition(targetFrame),
{}, this.getPosition(targetFrame),
{
attributes: this.interpolateAttributes(targetFrame),
attributes: this.getAttributes(targetFrame),
label: this.taskLabels[this.labelID],
group: this.group,
type: window.cvat.enums.ObjectType.TRACK,
@ -187,7 +187,7 @@
};
}
interpolateAttributes(targetFrame) {
getAttributes(targetFrame) {
const result = {};
// First of all copy all unmutable attributes
@ -226,6 +226,55 @@
return result;
}
getPosition(targetFrame) {
const {
leftFrame,
rightFrame,
} = this.neighborsFrames(targetFrame);
const rightPosition = Number.isInteger(rightFrame) ? this.shapes[rightFrame] : null;
const leftPosition = Number.isInteger(leftFrame) ? this.shapes[leftFrame] : null;
if (leftPosition && leftFrame === targetFrame) {
return {
points: [...leftPosition.points],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: leftPosition.zOrder,
};
}
if (rightPosition && leftPosition) {
return this.interpolatePosition(
leftPosition,
rightPosition,
targetFrame,
);
}
if (rightPosition) {
return {
points: [...rightPosition.points],
occluded: rightPosition.occluded,
outside: true,
zOrder: 0,
};
}
if (leftPosition) {
return {
points: [...leftPosition.points],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: 0,
};
}
throw new window.cvat.exceptions.ScriptingError(
`No one neightbour frame found for the track with client ID: "${this.id}"`,
);
}
}
class Tag extends Annotation {
@ -287,73 +336,408 @@
this.shape = window.cvat.enums.ObjectShape.RECTANGLE;
}
interpolatePosition(targetFrame) {
const {
leftFrame,
rightFrame,
} = this.neighborsFrames(targetFrame);
interpolatePosition(leftPosition, rightPosition, targetFrame) {
const offset = (targetFrame - leftPosition.frame) / (
rightPosition.frame - leftPosition.frame);
const positionOffset = [
rightPosition.points[0] - leftPosition.points[0],
rightPosition.points[1] - leftPosition.points[1],
rightPosition.points[2] - leftPosition.points[2],
rightPosition.points[3] - leftPosition.points[3],
];
return { // xtl, ytl, xbr, ybr
points: [
leftPosition.points[0] + positionOffset[0] * offset,
leftPosition.points[1] + positionOffset[1] * offset,
leftPosition.points[2] + positionOffset[2] * offset,
leftPosition.points[3] + positionOffset[3] * offset,
],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: leftPosition.zOrder,
};
}
}
class PolyTrack extends Track {
constructor(data, clientID, color, injection) {
super(data, clientID, color, injection);
}
const rightPosition = rightFrame ? this.shapes[rightFrame] : null;
const leftPosition = leftFrame ? this.shapes[leftFrame] : null;
interpolatePosition(leftPosition, rightPosition, targetFrame) {
function findBox(points) {
let xmin = Number.MAX_SAFE_INTEGER;
let ymin = Number.MAX_SAFE_INTEGER;
let xmax = Number.MIN_SAFE_INTEGER;
let ymax = Number.MIN_SAFE_INTEGER;
for (let i = 0; i < points.length; i += 2) {
if (points[i] < xmin) xmin = points[i];
if (points[i + 1] < ymin) ymin = points[i + 1];
if (points[i] > xmax) xmax = points[i];
if (points[i + 1] > ymax) ymax = points[i + 1];
}
if (leftPosition && leftFrame === targetFrame) {
return {
points: [...leftPosition.points],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: leftPosition.zOrder,
xmin,
ymin,
xmax,
ymax,
};
}
if (rightPosition && leftPosition) {
const offset = (targetFrame - leftFrame) / (rightPosition - leftPosition);
const positionOffset = [
rightPosition.points[0] - leftPosition.points[0],
rightPosition.points[1] - leftPosition.points[1],
rightPosition.points[2] - leftPosition.points[2],
rightPosition.points[3] - leftPosition.points[3],
];
return { // xtl, ytl, xbr, ybr
points: [
leftPosition.points[0] + positionOffset[0] * offset,
leftPosition.points[1] + positionOffset[1] * offset,
leftPosition.points[2] + positionOffset[2] * offset,
leftPosition.points[3] + positionOffset[3] * offset,
],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: leftPosition.zOrder,
};
function normalize(points, box) {
const normalized = [];
const width = box.xmax - box.xmin;
const height = box.ymax - box.ymin;
for (let i = 0; i < points.length; i += 2) {
normalized.push(
(points[i] - box.xmin) / width,
(points[i + 1] - box.ymin) / height,
);
}
return normalized;
}
if (rightPosition) {
return {
points: [...rightPosition.points],
occluded: rightPosition.occluded,
outside: true,
zOrder: 0,
};
function denormalize(points, box) {
const denormalized = [];
const width = box.xmax - box.xmin;
const height = box.ymax - box.ymin;
for (let i = 0; i < points.length; i += 2) {
denormalized.push(
points[i] * width + box.xmin,
points[i + 1] * height + box.ymin,
);
}
return denormalized;
}
if (leftPosition) {
return {
points: [...leftPosition.points],
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: 0,
function toPoints(array) {
const points = [];
for (let i = 0; i < array.length; i += 2) {
points.push({
x: array[i],
y: array[i + 1],
});
}
return points;
}
function toArray(points) {
const array = [];
for (const point of points) {
array.push(point.x, point.y);
}
return array;
}
function computeDistances(source, target) {
const distances = {};
for (let i = 0; i < source.length; i++) {
distances[i] = distances[i] || {};
for (let j = 0; j < target.length; j++) {
const dx = source[i].x - target[j].x;
const dy = source[i].y - target[j].y;
distances[i][j] = Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2));
}
}
return distances;
}
function truncateByThreshold(mapping, threshold) {
for (const key of Object.keys(mapping)) {
if (mapping[key].distance > threshold) {
delete mapping[key];
}
}
}
// https://en.wikipedia.org/wiki/Stable_marriage_problem
// TODO: One of important part of the algorithm is to correctly match
// "corner" points. Thus it is possible for each of such point calculate
// a descriptor (d) and use (x, y, d) to calculate the distance. One more
// idea is to be sure that order or matched points is preserved. For example,
// if p1 matches q1 and p2 matches q2 and between p1 and p2 we don't have any
// points thus we should not have points between q1 and q2 as well.
function stableMarriageProblem(men, women, distances) {
const menPreferences = {};
for (const man of men) {
menPreferences[man] = women.concat()
.sort((w1, w2) => distances[man][w1] - distances[man][w2]);
}
// Start alghoritm with max N^2 complexity
const womenMaybe = {}; // id woman:id man,distance
const menBusy = {}; // id man:boolean
let prefIndex = 0;
// While there is at least one free man
while (Object.values(menBusy).length !== men.length) {
// Every man makes offer to the best woman
for (const man of men) {
// The man have already found a woman
if (menBusy[man]) {
continue;
}
const woman = menPreferences[man][prefIndex];
const distance = distances[man][woman];
// A women chooses the best offer and says "maybe"
if (woman in womenMaybe && womenMaybe[woman].distance > distance) {
// A woman got better offer
const prevChoice = womenMaybe[woman].value;
delete womenMaybe[woman];
delete menBusy[prevChoice];
}
if (!(woman in womenMaybe)) {
womenMaybe[woman] = {
value: man,
distance,
};
menBusy[man] = true;
}
}
prefIndex++;
}
const result = {};
for (const woman of Object.keys(womenMaybe)) {
result[womenMaybe[woman].value] = {
value: woman,
distance: womenMaybe[woman].distance,
};
}
return result;
}
function getMapping(source, target) {
function sumEdges(points) {
let result = 0;
for (let i = 1; i < points.length; i += 2) {
const distance = Math.sqrt(Math.pow(points[i].x - points[i - 1].x, 2)
+ Math.pow(points[i].y - points[i - 1].y, 2));
result += distance;
}
// Corner case when work with one point
// Mapping in this case can't be wrong
if (!result) {
return Number.MAX_SAFE_INTEGER;
}
return result;
}
function computeDeviation(points, average) {
let result = 0;
for (let i = 1; i < points.length; i += 2) {
const distance = Math.sqrt(Math.pow(points[i].x - points[i - 1].x, 2)
+ Math.pow(points[i].y - points[i - 1].y, 2));
result += Math.pow(distance - average, 2);
}
return result;
}
const processedSource = [];
const processedTarget = [];
const distances = computeDistances(source, target);
const mapping = stableMarriageProblem(Array.from(source.keys()),
Array.from(target.keys()), distances);
const average = (sumEdges(target)
+ sumEdges(source)) / (target.length + source.length);
const meanSquareDeviation = Math.sqrt((computeDeviation(source, average)
+ computeDeviation(target, average)) / (source.length + target.length));
const threshold = average + 3 * meanSquareDeviation; // 3 sigma rule
truncateByThreshold(mapping, threshold);
for (const key of Object.keys(mapping)) {
mapping[key] = mapping[key].value;
}
// const receivingOrder = Object.keys(mapping).map(x => +x).sort((a,b) => a - b);
const receivingOrder = this.appendMapping(mapping, source, target);
for (const pointIdx of receivingOrder) {
processedSource.push(source[pointIdx]);
processedTarget.push(target[mapping[pointIdx]]);
}
return [processedSource, processedTarget];
}
let leftBox = findBox(leftPosition.points);
let rightBox = findBox(rightPosition.points);
// Sometimes (if shape has one point or shape is line),
// We can get box with zero area
// Next computation will be with NaN in this case
// We have to prevent it
const delta = 1;
if (leftBox.xmax - leftBox.xmin < delta || rightBox.ymax - rightBox.ymin < delta) {
leftBox = {
xmin: 0,
xmax: 1024, // TODO: Get actual image size
ymin: 0,
ymax: 768,
};
rightBox = leftBox;
}
throw new window.cvat.exceptions.ScriptingError(
`No one neightbour frame found for the track with client ID: "${this.id}"`,
);
const leftPoints = toPoints(normalize(leftPosition.points, leftBox));
const rightPoints = toPoints(normalize(rightPosition.points, rightBox));
let newLeftPoints = [];
let newRightPoints = [];
if (leftPoints.length > rightPoints.length) {
const [
processedRight,
processedLeft,
] = getMapping.call(this, rightPoints, leftPoints);
newLeftPoints = processedLeft;
newRightPoints = processedRight;
} else {
const [
processedLeft,
processedRight,
] = getMapping.call(this, leftPoints, rightPoints);
newLeftPoints = processedLeft;
newRightPoints = processedRight;
}
const absoluteLeftPoints = denormalize(toArray(newLeftPoints), leftBox);
const absoluteRightPoints = denormalize(toArray(newRightPoints), rightBox);
const offset = (targetFrame - leftPosition.frame) / (
rightPosition.frame - leftPosition.frame);
const interpolation = [];
for (let i = 0; i < absoluteLeftPoints.length; i++) {
interpolation.push(absoluteLeftPoints[i] + (
absoluteRightPoints[i] - absoluteLeftPoints[i]) * offset);
}
return {
points: interpolation,
occluded: leftPosition.occluded,
outside: leftPosition.outside,
zOrder: leftPosition.zOrder,
};
}
}
class PolyTrack extends Track {
constructor(data, clientID, color, injection) {
super(data, clientID, color, injection);
// mapping is predicted order of points sourse_idx:target_idx
// some points from source and target can absent in mapping
// source, target - arrays of points. Target array size >= sourse array size
appendMapping(mapping, source, target) {
const targetMatched = Object.values(mapping).map(x => +x);
const sourceMatched = Object.keys(mapping).map(x => +x);
const orderForReceive = [];
function findNeighbors(point) {
let prev = point;
let next = point;
if (!targetMatched.length) {
// Prevent infinity loop
throw window.cvat.exceptions.ScriptingError('Interpolation mapping is empty');
}
while (!targetMatched.includes(prev)) {
prev--;
if (prev < 0) {
prev = target.length - 1;
}
}
while (!targetMatched.includes(next)) {
next++;
if (next >= target.length) {
next = 0;
}
}
return [prev, next];
}
function computeOffset(point, prev, next) {
const pathPoints = [];
while (prev !== next) {
pathPoints.push(target[prev]);
prev++;
if (prev >= target.length) {
prev = 0;
}
}
pathPoints.push(target[next]);
let curveLength = 0;
let offset = 0;
let iCrossed = false;
for (let k = 1; k < pathPoints.length; k++) {
const p1 = pathPoints[k];
const p2 = pathPoints[k - 1];
const distance = Math.sqrt(Math.pow(p1.x - p2.x, 2) + Math.pow(p1.y - p2.y, 2));
if (!iCrossed) {
offset += distance;
}
curveLength += distance;
if (target[point] === pathPoints[k]) {
iCrossed = true;
}
}
if (!curveLength) {
return 0;
}
return offset / curveLength;
}
for (let i = 0; i < target.length; i++) {
const index = targetMatched.indexOf(i);
if (index === -1) {
// We have to find a neighbours which have been mapped
const [prev, next] = findNeighbors(i);
// Now compute edge offset
const offset = computeOffset(i, prev, next);
// Get point between two neighbors points
const prevPoint = target[prev];
const nextPoint = target[next];
const autoPoint = {
x: prevPoint.x + (nextPoint.x - prevPoint.x) * offset,
y: prevPoint.y + (nextPoint.y - prevPoint.y) * offset,
};
// Put it into matched
source.push(autoPoint);
mapping[source.length - 1] = i;
orderForReceive.push(source.length - 1);
} else {
orderForReceive.push(sourceMatched[index]);
}
}
return orderForReceive;
}
}
@ -369,6 +753,10 @@
super(data, clientID, color, injection);
this.shape = window.cvat.enums.ObjectShape.POLYLINE;
}
appendMapping(leftRightMapping, leftPoints, rightPoints) {
// TODO after checking how it works with polygons
}
}
class PointsTrack extends PolyTrack {

Write Preview
Loading…
Cancel
Save