Added capacity for multiple rectangle labels; bug fixes

11 files changed, 385 insertions, 46 deletions

diff --git a/Makefile b/Makefile
index d6032e3..de8b75b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 LIBS=libconfuse opencv4
-override CXXFLAGS += -c -Wall -fPIC -std=c++20 -Wno-deprecated-enum-enum-conversion `pkg-config $(LIBS) --cflags`
+override CXXFLAGS += -MMD -Wall -std=c++20 -Wno-deprecated-enum-enum-conversion `pkg-config $(LIBS) --cflags`
 override LDFLAGS += `pkg-config $(LIBS) --libs`
 SOURCES=$(wildcard src/*.cc) 
 OBJECTS=$(SOURCES:.cc=.o)
diff --git a/makeCrop.py b/makeCrop.py
index 8865dad..19ce369 100755
--- a/makeCrop.py
+++ b/makeCrop.py
@@ -1,22 +1,21 @@
 #!/usr/bin/env python3
-# PYTHON_ARGCOMPLETE_OK
 
-import argparse, argcomplete
+import argparse
 
-parser = argparse.ArgumentParser(description='Crop and scale a video based on bounding boxes')
+parser = argparse.ArgumentParser(description='Crop and scale a video based on bounding boxes', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument('boxes', help='Path to csv file where bounding boxes are saved')
 parser.add_argument('video', help='Path to video to crop')
 parser.add_argument('--intermediary', help='Path to intermediary video if size differs')
+parser.add_argument('--no-interpolate', help='Do not interpolate over gaps between box updates', action='store_true')
 parser.add_argument('--label', help='Boxes label (uses all if not provided)')
-parser.add_argument('--save-stem', help='Location to save stem.npz and stem.avi. Default: out', default='out')
+parser.add_argument('--save-stem', help='Location to save stem.npz and stem.avi', default='out')
 parser.add_argument('--skip-zero-boxes', help='Skip boxes with zero size', action='store_true')
+parser.add_argument('--outputWH', help='Output width and height', nargs=2, default=[64,64], type=int)
 parser.add_argument('--debug', help='Save a debugging video as well', action='store_true')
 
-argcomplete.autocomplete(parser)
 args = parser.parse_args()
 
-OUTW = 64
-OUTH = 64
+OUTW, OUTH = args.outputWH
 
 import csv
 with open(args.boxes) as f:
@@ -47,6 +46,51 @@ boxes = [[b[0], float(b[1]), int(int(b[2]) * zoomX), int(int(b[3]) * zoomY), int
 if args.skip_zero_boxes:
     boxes = [b for b in boxes if b[2] != b[4] and b[3] != b[5]]
 
+def alignDims(box: list, whRatio: float = 1.0) -> list:
+    '''Make a bounding box dimentions adhere to given ratio
+
+    Arguments:
+      box: The bounding box, formatted [x1, y1, x2, y2]
+      whRatio: The desired ratio of width to height
+
+    Returns:
+      list: The adjusted box
+    '''
+    coords1, coords2 = [0, 2], [1, 3]
+    if (box[coords2[1]] - box[coords2[0]]) * whRatio > (box[coords1[1]] - box[coords1[0]]):
+        coords1, coords2 = coords2, coords1
+        whRatio = 1 / whRatio
+    # coords1 > whRatio * coords2, but we want them equal
+    d = int((box[coords1[1]] - box[coords1[0]]) / whRatio - (box[coords2[1]] - box[coords2[0]]))
+    box[coords2[0]] -= int(d/2)
+    box[coords2[1]] += int(d/2) + (d % 2)
+    return box
+
+def shift(box: list, w: int, h: int) -> list:
+    '''Shift a bounding box to be within w,h bounds
+
+    Arguments:
+      box: The bounding box, formatted [x1, y1, x2, y2]
+      w: The width of the frame (i.e., maximum x value)
+      h: The height of the frame (i.e., maximum y value)
+
+    Returns:
+      list: The shifted box
+    '''
+    bounds = [[0, int(w)], [0, int(h)]]
+    coords = [[0, 2], [1, 3]]
+    for b, c in zip(bounds, coords):
+        s = 0
+        if box[c[0]] < b[0]:
+            s = -box[c[0]]
+        elif box[c[1]] > b[1]:
+            s = b[1] - box[c[1]]
+        box[c[0]] += s
+        box[c[1]] += s
+    return box
+
+boxes = [box[0:2] + shift(alignDims(box[2:], OUTW/OUTH), width, height) for box in boxes]
+
 import numpy as np
 outArry = []
 outVid = cv2.VideoWriter(args.save_stem + '.avi', cv2.VideoWriter_fourcc('M','J','P','G'), fps, (OUTW, OUTH))
@@ -67,8 +111,14 @@ while cap.isOpened():
     # Make boxes[0] the most recent timestamp without going over current time
     while len(boxes) >= 2 and boxes[1][1] <= time:
         boxes = boxes[1:]
-    x1, x2 = sorted([boxes[0][2], boxes[0][4]])
-    y1, y2 = sorted([boxes[0][3], boxes[0][5]])
+    if len(boxes) == 1: # Stop when we run out
+        break
+    box = boxes[0]
+    if not args.no_interpolate:
+        weight = (boxes[1][1] - time) / (boxes[1][1] - boxes[0][1])
+        box = [boxes[0][0], time] + [int(boxes[0][i] * weight + boxes[1][i] * (1-weight)) for i in range(2, 6)]
+    x1, x2 = sorted([box[2], box[4]])
+    y1, y2 = sorted([box[3], box[5]])
     cropped = frame[y1:y2, x1:x2]
     if cropped.size < 1:
         resized = np.zeros((OUTH, OUTW, 3), dtype=cropped.dtype)
@@ -88,5 +138,7 @@ while cap.isOpened():
 
 bar.finish()
 
+print(f'Ending at time = {time}')
+
 # Save out array
 np.savez_compressed(args.save_stem + '.npz', np.array(outArry))
diff --git a/readme.md b/readme.md
index b69a2fb..7ad1785 100644
--- a/readme.md
+++ b/readme.md
@@ -20,6 +20,7 @@ make
 Available labels are configured in `annotator.conf`. Controls are as follows:
 
  * Number keys: Make annotation
+ * c: cycle through rectangle labels
  * Escape: Quit (doesn't save)
  * Space: Pause
  * Left arrow: Go backward 1 frame
diff --git a/src/annotator.d b/src/annotator.d
new file mode 100644
index 0000000..6a9139a
--- /dev/null
+++ b/src/annotator.d
@@ -0,0 +1 @@
+src/annotator.o: src/annotator.cc src/playback.h src/ui.h src/labeller.h
diff --git a/src/labeller.d b/src/labeller.d
new file mode 100644
index 0000000..615ef8c
--- /dev/null
+++ b/src/labeller.d
@@ -0,0 +1 @@
+src/labeller.o: src/labeller.cc src/labeller.h src/settings.h
diff --git a/src/playback.cc b/src/playback.cc
index 700567a..3203f47 100644
--- a/src/playback.cc
+++ b/src/playback.cc
@@ -5,13 +5,15 @@
 #include <stdexcept>
 #include <iostream>
 #include <thread>
+#include <map>
 
 namespace frontend {
     struct playback_impl {
         std::vector<cv::Mat> images;
-        int fps;
+        double fps;
         double reportedDuration;
         std::size_t frameNum = 0;
+        std::map<std::string, cv::Scalar> names2colors;
     };
 
     playback::playback(const std::filesystem::path& video, std::size_t frameCap) {
@@ -29,14 +31,21 @@ namespace frontend {
         }
         data->reportedDuration = cap.get(cv::CAP_PROP_POS_MSEC) / 1000.0;
         std::cout << "Reported duration: " << data->reportedDuration << " seconds" << std::endl;
-        cap.release();
     }
 
-    void playback::display(const std::string& windowName, const backend::rect &r) const {
-        cv::Mat curr = data->images[data->frameNum].clone();
-        if(r.nonzero()) {
-            // Update with the rectangle
-            cv::rectangle(curr, cv::Point(r.x1, r.y1), cv::Point(r.x2, r.y2), cv::Scalar(255, 0, 0), 2, cv::LINE_8);
+    void playback::display(const std::string& windowName, const std::vector<backend::label>& labels) const {
+        auto curr = data->images[data->frameNum].clone();
+        std::vector<cv::Scalar> colors = {cv::Scalar(255, 0, 0), cv::Scalar(0, 255, 0), cv::Scalar(0, 0, 255), cv::Scalar(255, 255, 0), cv::Scalar(255, 0, 255), cv::Scalar(0, 255, 255), cv::Scalar(255, 255, 255)};
+        for(auto l : labels) {
+            if(! data->names2colors.contains(l.name)) {
+                data->names2colors.insert({l.name, colors[data->names2colors.size()%colors.size()]});
+            }
+            auto color = data->names2colors.at(l.name);
+            auto r = l.location;
+            if(r.nonzero()) {
+                // Update with the rectangle
+                cv::rectangle(curr, cv::Point(r.x1, r.y1), cv::Point(r.x2, r.y2), color, 2, cv::LINE_8);
+            }
         }
         cv::imshow(windowName, curr);
     }
@@ -58,11 +67,11 @@ namespace frontend {
     }
 
     double playback::getTime() const {
-        return getFrame() / (double) data->fps;
+        return getFrame() / data->fps;
     }
     
     void playback::interFrameSleep() const {
-        std::this_thread::sleep_for(std::chrono::milliseconds(1000/data->fps));
+        std::this_thread::sleep_for(std::chrono::milliseconds(int(1000/data->fps)));
     }
 
     std::size_t playback::getMaxFrame() const {
@@ -70,6 +79,9 @@ namespace frontend {
     }
 
     double playback::getMaxTime() const {
-        return getMaxFrame() / (double) data->fps;
+        return getMaxFrame() / data->fps;
+    }
+
+    void playback::release() const {
     }
 }
diff --git a/src/playback.d b/src/playback.d
new file mode 100644
index 0000000..442c700
--- /dev/null
+++ b/src/playback.d
@@ -0,0 +1,112 @@
+src/playback.o: src/playback.cc src/playback.h src/labeller.h \
+ /usr/include/opencv4/opencv2/opencv.hpp \
+ /usr/include/opencv4/opencv2/opencv_modules.hpp \
+ /usr/include/opencv4/opencv2/core.hpp \
+ /usr/include/opencv4/opencv2/core/cvdef.h \
+ /usr/include/opencv4/opencv2/core/version.hpp \
+ /usr/include/opencv4/opencv2/core/hal/interface.h \
+ /usr/include/opencv4/opencv2/core/cv_cpu_dispatch.h \
+ /usr/include/opencv4/opencv2/core/base.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd_wrapper.hpp \
+ /usr/include/opencv4/opencv2/core/neon_utils.hpp \
+ /usr/include/opencv4/opencv2/core/vsx_utils.hpp \
+ /usr/include/opencv4/opencv2/core/check.hpp \
+ /usr/include/opencv4/opencv2/core/traits.hpp \
+ /usr/include/opencv4/opencv2/core/matx.hpp \
+ /usr/include/opencv4/opencv2/core/saturate.hpp \
+ /usr/include/opencv4/opencv2/core/fast_math.hpp \
+ /usr/include/opencv4/opencv2/core/types.hpp \
+ /usr/include/opencv4/opencv2/core/mat.hpp \
+ /usr/include/opencv4/opencv2/core/bufferpool.hpp \
+ /usr/include/opencv4/opencv2/core/mat.inl.hpp \
+ /usr/include/opencv4/opencv2/core/persistence.hpp \
+ /usr/include/opencv4/opencv2/core/operations.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd.inl.hpp \
+ /usr/include/opencv4/opencv2/core/utility.hpp \
+ /usr/include/opencv4/opencv2/core/optim.hpp \
+ /usr/include/opencv4/opencv2/core/ovx.hpp \
+ /usr/include/opencv4/opencv2/core/cvdef.h \
+ /usr/include/opencv4/opencv2/calib3d.hpp \
+ /usr/include/opencv4/opencv2/features2d.hpp \
+ /usr/include/opencv4/opencv2/flann/miniflann.hpp \
+ /usr/include/opencv4/opencv2/flann/defines.h \
+ /usr/include/opencv4/opencv2/flann/config.h \
+ /usr/include/opencv4/opencv2/core/affine.hpp \
+ /usr/include/opencv4/opencv2/dnn.hpp \
+ /usr/include/opencv4/opencv2/dnn/dnn.hpp \
+ /usr/include/opencv4/opencv2/core/async.hpp \
+ /usr/include/opencv4/opencv2/dnn/../dnn/version.hpp \
+ /usr/include/opencv4/opencv2/dnn/dict.hpp \
+ /usr/include/opencv4/opencv2/dnn/layer.hpp \
+ /usr/include/opencv4/opencv2/dnn/dnn.inl.hpp \
+ /usr/include/opencv4/opencv2/dnn/utils/inference_engine.hpp \
+ /usr/include/opencv4/opencv2/dnn/utils/../dnn.hpp \
+ /usr/include/opencv4/opencv2/flann.hpp \
+ /usr/include/opencv4/opencv2/flann/flann_base.hpp \
+ /usr/include/opencv4/opencv2/flann/general.h \
+ /usr/include/opencv4/opencv2/flann/matrix.h \
+ /usr/include/opencv4/opencv2/flann/params.h \
+ /usr/include/opencv4/opencv2/flann/any.h \
+ /usr/include/opencv4/opencv2/flann/defines.h \
+ /usr/include/opencv4/opencv2/flann/saving.h \
+ /usr/include/opencv4/opencv2/flann/nn_index.h \
+ /usr/include/opencv4/opencv2/flann/result_set.h \
+ /usr/include/opencv4/opencv2/flann/all_indices.h \
+ /usr/include/opencv4/opencv2/flann/kdtree_index.h \
+ /usr/include/opencv4/opencv2/flann/dynamic_bitset.h \
+ /usr/include/opencv4/opencv2/flann/dist.h \
+ /usr/include/opencv4/opencv2/flann/heap.h \
+ /usr/include/opencv4/opencv2/flann/allocator.h \
+ /usr/include/opencv4/opencv2/flann/random.h \
+ /usr/include/opencv4/opencv2/flann/kdtree_single_index.h \
+ /usr/include/opencv4/opencv2/flann/kmeans_index.h \
+ /usr/include/opencv4/opencv2/flann/logger.h \
+ /usr/include/opencv4/opencv2/flann/composite_index.h \
+ /usr/include/opencv4/opencv2/flann/linear_index.h \
+ /usr/include/opencv4/opencv2/flann/hierarchical_clustering_index.h \
+ /usr/include/opencv4/opencv2/flann/lsh_index.h \
+ /usr/include/opencv4/opencv2/flann/lsh_table.h \
+ /usr/include/opencv4/opencv2/flann/autotuned_index.h \
+ /usr/include/opencv4/opencv2/flann/ground_truth.h \
+ /usr/include/opencv4/opencv2/flann/index_testing.h \
+ /usr/include/opencv4/opencv2/flann/timer.h \
+ /usr/include/opencv4/opencv2/flann/sampling.h \
+ /usr/include/opencv4/opencv2/highgui.hpp \
+ /usr/include/opencv4/opencv2/imgcodecs.hpp \
+ /usr/include/opencv4/opencv2/videoio.hpp \
+ /usr/include/opencv4/opencv2/imgproc.hpp \
+ /usr/include/opencv4/opencv2/./imgproc/segmentation.hpp \
+ /usr/include/opencv4/opencv2/ml.hpp \
+ /usr/include/opencv4/opencv2/ml/ml.inl.hpp \
+ /usr/include/opencv4/opencv2/objdetect.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_dictionary.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_board.hpp \
+ /usr/include/opencv4/opencv2/objdetect/graphical_code_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/detection_based_tracker.hpp \
+ /usr/include/opencv4/opencv2/objdetect/face.hpp \
+ /usr/include/opencv4/opencv2/objdetect/charuco_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/barcode.hpp \
+ /usr/include/opencv4/opencv2/photo.hpp \
+ /usr/include/opencv4/opencv2/stitching.hpp \
+ /usr/include/opencv4/opencv2/stitching/warpers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers.hpp \
+ /usr/include/opencv4/opencv2/core/cuda.hpp \
+ /usr/include/opencv4/opencv2/core/cuda_types.hpp \
+ /usr/include/opencv4/opencv2/core/cuda.inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers_inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/matchers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/motion_estimators.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/matchers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/util.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/util_inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/camera.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/exposure_compensate.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/seam_finders.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/blenders.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/camera.hpp \
+ /usr/include/opencv4/opencv2/video.hpp \
+ /usr/include/opencv4/opencv2/video/tracking.hpp \
+ /usr/include/opencv4/opencv2/video/background_segm.hpp
diff --git a/src/playback.h b/src/playback.h
index 3f6e67f..363a360 100644
--- a/src/playback.h
+++ b/src/playback.h
@@ -2,9 +2,10 @@
 #include <filesystem>
 #include <memory>
 #include <string>
+#include <vector>
 
 namespace backend {
-    struct rect;
+    struct label;
 }
 
 namespace frontend {
@@ -12,7 +13,7 @@ namespace frontend {
     class playback {
         public:
             playback(const std::filesystem::path& video, std::size_t frameCap=0);
-            void display(const std::string& windowName, const backend::rect &r) const;
+            void display(const std::string& windowName, const std::vector<backend::label>& labels) const;
             bool seekFrame(std::size_t frameNum);
             std::size_t getFrame() const;
             bool seekTime(double time);
@@ -20,6 +21,7 @@ namespace frontend {
             void interFrameSleep() const;
             std::size_t getMaxFrame() const;
             double getMaxTime() const;
+            void release() const;
 
         private:
             std::shared_ptr<playback_impl> data;
diff --git a/src/settings.d b/src/settings.d
new file mode 100644
index 0000000..5ad3773
--- /dev/null
+++ b/src/settings.d
@@ -0,0 +1 @@
+src/settings.o: src/settings.cc src/settings.h
diff --git a/src/ui.cc b/src/ui.cc
index 3940955..541444a 100644
--- a/src/ui.cc
+++ b/src/ui.cc
@@ -12,12 +12,64 @@ namespace frontend {
         std::shared_ptr<backend::labeller> llr;
         bool play = false;
         bool stalePrintout = true;
+        int rectIndex = -1;
     };
 
     ui::ui(std::shared_ptr<playback> playback, std::shared_ptr<backend::labeller> labeller) {
         data = std::shared_ptr<ui_impl>(new ui_impl);
         data->pb = playback;
         data->llr = labeller;
+        if(!data->llr->getRectangleLabels().empty()) {
+            data->rectIndex = 0;
+        }
+    }
+
+    std::string getCurrentRectName(ui_impl* data) {
+        std::string name = "Rectangle";
+        if(data->rectIndex >= 0) {
+            name = data->llr->getRectangleLabels()[data->rectIndex];
+        }
+        return name;
+    }
+
+    std::pair<backend::label*, backend::label*> getSurroundingLabels(ui_impl* data, std::string name) {
+        std::pair<backend::label*, backend::label*> ret = {nullptr, nullptr};
+        auto &labs = data->llr->getEditableLabels();
+        for(auto it = labs.begin(); it != labs.end(); it++) {
+            if(it->name != name) {
+                continue;
+            }
+            if(it->time <= data->pb->getTime() && (!ret.first || ret.first->time < it->time)) {
+                ret.first = &(*it);
+            }
+            if(it->time > data->pb->getTime() && (!ret.second || ret.second->time > it->time)) {
+                ret.second = &(*it);
+            }
+        }
+        return ret;
+    }
+
+    std::vector<backend::label> getInterpolatedRectLabels(ui_impl* data) {
+        std::vector<backend::label> ret;
+        for(auto name : data->llr->getRectangleLabels()) {
+            auto p = getSurroundingLabels(data, name);
+            if(p.first && ! p.second) {
+                ret.push_back(*p.first);
+            } else if(p.first && p.second) {
+                backend::label l(p.first->name, data->pb->getTime());
+                double weight = (p.second->time-l.time)/(p.second->time-p.first->time);
+                l.location.x1 = weight*p.first->location.x1 + (1-weight)*p.second->location.x1;
+                l.location.x2 = weight*p.first->location.x2 + (1-weight)*p.second->location.x2;
+                l.location.y1 = weight*p.first->location.y1 + (1-weight)*p.second->location.y1;
+                l.location.y2 = weight*p.first->location.y2 + (1-weight)*p.second->location.y2;
+                ret.push_back(l);
+            }
+        }
+        return ret;
+    }
+
+    backend::label* getCurrentRectLabel(ui_impl* data) {
+        return getSurroundingLabels(data, getCurrentRectName(data)).first;
     }
 
     bool handleKey(int keycode, const std::shared_ptr<ui_impl>& data) {
@@ -31,6 +83,7 @@ namespace frontend {
             } else {
                 switch(keycode) {
                     case 27: // Escape quits
+                        data->pb->release();
                         std::cout << std::endl;
                         return true;
                     case 32: // Space pauses
@@ -68,6 +121,12 @@ namespace frontend {
                     case 115: // s saves
                         data->llr->save();
                         break;
+                    case 99: // c cycles rect labels
+                        if(!data->llr->getRectangleLabels().empty()) {
+                            data->rectIndex = (data->rectIndex + 1) % data->llr->getRectangleLabels().size();
+                        }
+                        std::cout << "Set rect name to " << getCurrentRectName(data.get()) << std::endl;
+                        break;
                     default:
                         std::cout << "Pressed the " << keycode << " key" << std::endl;
                         break;
@@ -77,16 +136,6 @@ namespace frontend {
         return false;
     }
 
-    backend::label* getCurrentRectLabel(ui_impl *data) {
-        auto &labs = data->llr->getEditableLabels();
-        for(auto rit = labs.rbegin(); rit != labs.rend(); rit++) {
-            if(rit->location.nonzero() && rit->time <= data->pb->getTime()) {
-                return &(*rit);
-            }
-        }
-        return nullptr;
-    }
-
     void mouseCallback(int event, int x, int y, int flags, void* userdata) {
         // userdata is actually a &shared_ptr<ui_impl>
         auto data = static_cast<ui_impl*>(userdata);
@@ -100,14 +149,7 @@ namespace frontend {
             if(lab && lab->time == time) {
                 // Use this one
             } else { // Make one
-                // Try to use the first rectangle annotation, fall back on normal, then "Rectangle"
-                std::string name = "Rectangle";
-                if(!data->llr->getRectangleLabels().empty()) {
-                    name = data->llr->getRectangleLabels()[0];
-                } else if(!data->llr->getLabels().empty()) {
-                    name = data->llr->getLabels()[0];
-                }
-                data->llr->applyLabel(name, time);
+                data->llr->applyLabel(getCurrentRectName(data), time);
                 lab = &data->llr->getEditableLabels().back();
             }
             lab->location.x1 = lab->location.x2 = x;
@@ -118,10 +160,13 @@ namespace frontend {
         } else if(event == cv::EVENT_MOUSEMOVE) {
             // If the mouse is down, update x2 and y2 of current window
             if(mouseDown) {
-                auto lab = getCurrentRectLabel(data); // It will be valid! TODO: Will it?
-                //std::cout << "Updating location from " << lab->location.x2 << ", " << lab->location.y2 << " to " << x << ", " << y << "\n";
-                lab->location.x2 = x;
-                lab->location.y2 = y;
+                auto lab = getCurrentRectLabel(data);
+                if(lab) {
+                    lab->location.x2 = x;
+                    lab->location.y2 = y;
+                } else {
+                    std::cout << "ERROR: Invalid label!" << std::endl;
+                }
             }
         }
     }
@@ -138,10 +183,10 @@ namespace frontend {
         ioctl(STDOUT_FILENO, TIOCGWINSZ, &size);
         int cols = size.ws_col;
         // Register mouse callback
-        data->pb->display("Video", (getCurrentRectLabel(data.get()))? getCurrentRectLabel(data.get())->location : backend::rect());
+        data->pb->display("Video", getInterpolatedRectLabels(data.get()));
         cv::setMouseCallback("Video", mouseCallback, data.get());
         while(true) {
-            data->pb->display("Video", (getCurrentRectLabel(data.get()))? getCurrentRectLabel(data.get())->location : backend::rect());
+            data->pb->display("Video", getInterpolatedRectLabels(data.get()));
             data->pb->interFrameSleep();
             if(handleKey(cv::pollKey(), data)) {
                 break;
diff --git a/src/ui.d b/src/ui.d
new file mode 100644
index 0000000..3b6d87a
--- /dev/null
+++ b/src/ui.d
@@ -0,0 +1,112 @@
+src/ui.o: src/ui.cc src/ui.h src/playback.h src/labeller.h \
+ /usr/include/opencv4/opencv2/opencv.hpp \
+ /usr/include/opencv4/opencv2/opencv_modules.hpp \
+ /usr/include/opencv4/opencv2/core.hpp \
+ /usr/include/opencv4/opencv2/core/cvdef.h \
+ /usr/include/opencv4/opencv2/core/version.hpp \
+ /usr/include/opencv4/opencv2/core/hal/interface.h \
+ /usr/include/opencv4/opencv2/core/cv_cpu_dispatch.h \
+ /usr/include/opencv4/opencv2/core/base.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd_wrapper.hpp \
+ /usr/include/opencv4/opencv2/core/neon_utils.hpp \
+ /usr/include/opencv4/opencv2/core/vsx_utils.hpp \
+ /usr/include/opencv4/opencv2/core/check.hpp \
+ /usr/include/opencv4/opencv2/core/traits.hpp \
+ /usr/include/opencv4/opencv2/core/matx.hpp \
+ /usr/include/opencv4/opencv2/core/saturate.hpp \
+ /usr/include/opencv4/opencv2/core/fast_math.hpp \
+ /usr/include/opencv4/opencv2/core/types.hpp \
+ /usr/include/opencv4/opencv2/core/mat.hpp \
+ /usr/include/opencv4/opencv2/core/bufferpool.hpp \
+ /usr/include/opencv4/opencv2/core/mat.inl.hpp \
+ /usr/include/opencv4/opencv2/core/persistence.hpp \
+ /usr/include/opencv4/opencv2/core/operations.hpp \
+ /usr/include/opencv4/opencv2/core/cvstd.inl.hpp \
+ /usr/include/opencv4/opencv2/core/utility.hpp \
+ /usr/include/opencv4/opencv2/core/optim.hpp \
+ /usr/include/opencv4/opencv2/core/ovx.hpp \
+ /usr/include/opencv4/opencv2/core/cvdef.h \
+ /usr/include/opencv4/opencv2/calib3d.hpp \
+ /usr/include/opencv4/opencv2/features2d.hpp \
+ /usr/include/opencv4/opencv2/flann/miniflann.hpp \
+ /usr/include/opencv4/opencv2/flann/defines.h \
+ /usr/include/opencv4/opencv2/flann/config.h \
+ /usr/include/opencv4/opencv2/core/affine.hpp \
+ /usr/include/opencv4/opencv2/dnn.hpp \
+ /usr/include/opencv4/opencv2/dnn/dnn.hpp \
+ /usr/include/opencv4/opencv2/core/async.hpp \
+ /usr/include/opencv4/opencv2/dnn/../dnn/version.hpp \
+ /usr/include/opencv4/opencv2/dnn/dict.hpp \
+ /usr/include/opencv4/opencv2/dnn/layer.hpp \
+ /usr/include/opencv4/opencv2/dnn/dnn.inl.hpp \
+ /usr/include/opencv4/opencv2/dnn/utils/inference_engine.hpp \
+ /usr/include/opencv4/opencv2/dnn/utils/../dnn.hpp \
+ /usr/include/opencv4/opencv2/flann.hpp \
+ /usr/include/opencv4/opencv2/flann/flann_base.hpp \
+ /usr/include/opencv4/opencv2/flann/general.h \
+ /usr/include/opencv4/opencv2/flann/matrix.h \
+ /usr/include/opencv4/opencv2/flann/params.h \
+ /usr/include/opencv4/opencv2/flann/any.h \
+ /usr/include/opencv4/opencv2/flann/defines.h \
+ /usr/include/opencv4/opencv2/flann/saving.h \
+ /usr/include/opencv4/opencv2/flann/nn_index.h \
+ /usr/include/opencv4/opencv2/flann/result_set.h \
+ /usr/include/opencv4/opencv2/flann/all_indices.h \
+ /usr/include/opencv4/opencv2/flann/kdtree_index.h \
+ /usr/include/opencv4/opencv2/flann/dynamic_bitset.h \
+ /usr/include/opencv4/opencv2/flann/dist.h \
+ /usr/include/opencv4/opencv2/flann/heap.h \
+ /usr/include/opencv4/opencv2/flann/allocator.h \
+ /usr/include/opencv4/opencv2/flann/random.h \
+ /usr/include/opencv4/opencv2/flann/kdtree_single_index.h \
+ /usr/include/opencv4/opencv2/flann/kmeans_index.h \
+ /usr/include/opencv4/opencv2/flann/logger.h \
+ /usr/include/opencv4/opencv2/flann/composite_index.h \
+ /usr/include/opencv4/opencv2/flann/linear_index.h \
+ /usr/include/opencv4/opencv2/flann/hierarchical_clustering_index.h \
+ /usr/include/opencv4/opencv2/flann/lsh_index.h \
+ /usr/include/opencv4/opencv2/flann/lsh_table.h \
+ /usr/include/opencv4/opencv2/flann/autotuned_index.h \
+ /usr/include/opencv4/opencv2/flann/ground_truth.h \
+ /usr/include/opencv4/opencv2/flann/index_testing.h \
+ /usr/include/opencv4/opencv2/flann/timer.h \
+ /usr/include/opencv4/opencv2/flann/sampling.h \
+ /usr/include/opencv4/opencv2/highgui.hpp \
+ /usr/include/opencv4/opencv2/imgcodecs.hpp \
+ /usr/include/opencv4/opencv2/videoio.hpp \
+ /usr/include/opencv4/opencv2/imgproc.hpp \
+ /usr/include/opencv4/opencv2/./imgproc/segmentation.hpp \
+ /usr/include/opencv4/opencv2/ml.hpp \
+ /usr/include/opencv4/opencv2/ml/ml.inl.hpp \
+ /usr/include/opencv4/opencv2/objdetect.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_dictionary.hpp \
+ /usr/include/opencv4/opencv2/objdetect/aruco_board.hpp \
+ /usr/include/opencv4/opencv2/objdetect/graphical_code_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/detection_based_tracker.hpp \
+ /usr/include/opencv4/opencv2/objdetect/face.hpp \
+ /usr/include/opencv4/opencv2/objdetect/charuco_detector.hpp \
+ /usr/include/opencv4/opencv2/objdetect/barcode.hpp \
+ /usr/include/opencv4/opencv2/photo.hpp \
+ /usr/include/opencv4/opencv2/stitching.hpp \
+ /usr/include/opencv4/opencv2/stitching/warpers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers.hpp \
+ /usr/include/opencv4/opencv2/core/cuda.hpp \
+ /usr/include/opencv4/opencv2/core/cuda_types.hpp \
+ /usr/include/opencv4/opencv2/core/cuda.inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers_inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/warpers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/matchers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/motion_estimators.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/matchers.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/util.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/util_inl.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/camera.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/exposure_compensate.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/seam_finders.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/blenders.hpp \
+ /usr/include/opencv4/opencv2/stitching/detail/camera.hpp \
+ /usr/include/opencv4/opencv2/video.hpp \
+ /usr/include/opencv4/opencv2/video/tracking.hpp \
+ /usr/include/opencv4/opencv2/video/background_segm.hpp