Commit 0519e50a authored by Thomas Dowrick's avatar Thomas Dowrick
Browse files

Merge branch 'crop-roi' into 'master'

Crop roi

See merge request WEISS/SoftwareRepositories/SNAPPY/scikit-surgery-davinci!5
parents 9ef39a05 4bb85ee4
Pipeline #1953 passed with stages
in 20 minutes and 20 seconds
......@@ -10,52 +10,47 @@ stages:
- deploy
build docs:
.build-install-template: &build-install-template
stage: build
except:
variables:
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]build?\]/i
script:
- tox -e docs
tags:
- shared-linux
- tox -e installer
artifacts:
paths:
- doc/
- dist/
expire_in: 1 week
build linux installer:
build docs:
stage: build
except:
variables:
- $CI_COMMIT_MESSAGE =~ /\[skip[ _-]build?\]/i
script:
- tox -e installer
- tox -e docs
tags:
- shared-linux
- shared-linux, gui
artifacts:
paths:
- dist/
- doc/
expire_in: 1 week
build linux installer:
<<: *build-install-template
tags:
- shared-linux, gui
build mac installer:
stage: build
script:
- tox -e installer
<<: *build-install-template
tags:
- shared-mac
artifacts:
paths:
- dist/
expire_in: 1 week
- shared-mac, gui
build windows installer:
stage: build
script:
- tox -e installer
<<: *build-install-template
tags:
- shared-win, gui
artifacts:
paths:
- dist/
expire_in: 1 week
test Linux:
stage: test
......
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
import logging
from sksurgerydavinci.ui.sksurgerydavinci_command_line import main
......
#coding=utf-8
""" AutoCropper Module. """
import numpy as np
class AutoCropBlackBorder:
""" Class to generate a ROI for an image based on a threshold value.
Originally developed for auto cropping video feed from the DaVinci Xi and
Tile Pro, where there is a central image (which we want to keep), surrounded
by a black region, and then other stuff that we're not interested in.
By searching for the start/end of the black region, we can auto crop the
bit we do want.
For speed, only the red channel of the image is scanned when detecting
black areas (See comment in set_roi for more details).
:param threshold: Threshold value
: param min_size: Minimum size (applies to both width and length)
of the expected output.
"""
def __init__(self, threshold=1, min_size=0):
self.min_size = min_size
self.threshold = threshold
self.roi = None
def get_bounds(self, vector):
""" Given a vector, skip any leading values that are > threshold,
then find the first contiguous range of values that are below the
threshold value.
If the whole vector is above the threshold, return the start/end
indexes.
e.g.
[0 0 0 50 50 50 0 0 0 ] return 3, 5 - [50 50 50]
[25 25 0 0 25 25 25 0 0] returns 4, 7 [25 25 25] as the leading 25s are
skipped.
[100 100 100 100 100 100] returns 0, 5.
:param vector: Input vector, in which to find the start/end bounds
:return start: First element that is above the threshold.
:return end: First element following start, that is below threshold.
"""
# If the image starts with a non black area, skip this. We want
# to start searching when the black border begins.
skip = 0
if vector[0] > self.threshold:
skip = np.argmax(vector < self.threshold)
start = skip + np.argmax(vector[skip:] > self.threshold)
end = start + self.min_size + \
np.argmax(vector[(start + self.min_size):] < self.threshold)
# HANDLE EDGE CASES
# argmax returns 0 if there is no match to the search criteria,
# which can give incorrect results, so it is necessary to
# identify some edge cases and act appropriately.
# print(f"skip: {skip}, start: {start}, end: {end}")
# There is a a border at the start, but not the end
if start == end and start != skip:
end = len(vector)
# There is a border at the end, but not at the start
if skip == start == end:
start = 0
# There is no border
if skip == start == end == 0:
end = len(vector)
return start, end
def get_roi(self, img):
""" Calculate the ROI.
Find the x/y extent of the ROI by averaging row and column values,
and then finding the area of interest."""
# Only use the red channel, otherwise we need to take the average along
# two dimensions, which is slow. This should be acceptable for the
# DaVinci.
col_mean = np.mean(img[:, :, 1], axis=(0))
row_mean = np.mean(img[:, :, 1], axis=(1))
start_y, end_y = self.get_bounds(row_mean)
start_x, end_x = self.get_bounds(col_mean)
self.roi = []
self.roi.append((start_x, start_y))
self.roi.append((end_x, end_y))
return self.roi
......@@ -3,6 +3,7 @@
# pylint: disable=fixme, attribute-defined-outside-init
# pylint: disable=no-name-in-module, too-many-public-methods
import os
from PySide2.QtCore import Qt, Signal
from PySide2.QtWidgets import QSlider, QVBoxLayout, QHBoxLayout, \
QLabel, QPushButton, \
......@@ -33,13 +34,15 @@ class UI(QWidget):
self.controls_vbox = QVBoxLayout()
self.opacity_box = QHBoxLayout()
self.visiblity_buttons = QHBoxLayout()
self.other_buttons = QHBoxLayout()
self.other_buttons = QVBoxLayout()
self.add_vtk_overlay_window(overlay_window)
self.add_opacity_slider()
self.layout.addLayout(self.visiblity_buttons)
self.layout.addLayout(self.other_buttons)
self.add_exit_button()
self.add_record_buttons()
self.add_crop_buttons()
self.screen = None
self.screen_geometry = None
......@@ -81,6 +84,28 @@ class UI(QWidget):
self.exit_button.setStyleSheet(style_sheet)
self.other_buttons.addWidget(self.exit_button)
def add_record_buttons(self):
""" Add buttons to take screenshot and record video.
Button callbacks should be implmenented in the Viewer class. """
self.screenshot_button = QPushButton('Take screenshot')
self.record_button = QPushButton('Record video')
self.record_buttons_layout = QHBoxLayout()
self.record_buttons_layout.addWidget(self.screenshot_button)
self.record_buttons_layout.addWidget(self.record_button)
self.other_buttons.addLayout(self.record_buttons_layout)
def add_crop_buttons(self):
""" Add buttons to crop the video stream. """
self.crop_buttons_layout = QHBoxLayout()
self.crop_button = QPushButton('Crop')
self.autocrop_button = QPushButton('Enable Auto Crop')
self.crop_buttons_layout.addWidget(self.crop_button)
self.crop_buttons_layout.addWidget(self.autocrop_button)
self.other_buttons.addLayout(self.crop_buttons_layout)
def on_exit_clicked(self):
""" Close the application when the exit button has been clicked"""
......
......@@ -7,7 +7,6 @@ import logging
from sksurgerydavinci import __version__
from sksurgerydavinci.ui.sksurgerydavinci_demo import run_demo
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)
def main(args=None):
......@@ -47,6 +46,13 @@ def main(args=None):
type=str,
help="Location of directory containing models to be loaded")
parser.add_argument(
"-l",
"--logging_debug",
action="store_true",
help="Enable Debug level logging"
)
version_string = __version__
friendly_version_string = version_string if version_string else 'unknown'
parser.add_argument(
......@@ -56,6 +62,13 @@ def main(args=None):
args = parser.parse_args(args)
if args.logging_debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
print(args)
run_demo(args)
\ No newline at end of file
......@@ -3,6 +3,7 @@
"""Hello world demo module"""
import sys
import vtk
from PySide2.QtWidgets import QApplication
from sksurgeryvtk.models.vtk_surface_model_directory_loader \
......@@ -68,5 +69,6 @@ def create_sample_model():
cone_actor.SetMapper(cone_mapper)
model.actor = cone_actor
model.set_name("Cone")
return [model]
""" Mono/Stereo viewing windows for Ardavinci"""
import logging
import datetime
from PySide2 import QtWidgets
from sksurgeryvtk.widgets.vtk_overlay_window import VTKOverlayWindow
from sksurgeryutils.common_overlay_apps import OverlayOnVideoFeed
from PySide2.QtCore import QTimer
from sksurgeryutils.common_overlay_apps import \
OverlayOnVideoFeedCropRecord, DuplicateOverlayWindow
from sksurgeryutils.utils.screen_utils import ScreenController
from sksurgerydavinci.ui.gui import UI
from sksurgerydavinci.davinci_xi_auto_cropping import AutoCropBlackBorder
LOGGER = logging.getLogger(__name__)
#pylint: disable= no-member, attribute-defined-outside-init
#pylint: disable=protected-access, no-self-use
#pylin: disable= no-member, attribute-defined-outside-init
#pylin: disable=protected-access, no-self-use
class DuplicateOverlayWindow(OverlayOnVideoFeed):
"""
Set the background of vtk_overlay_window to duplicate
that of another vtk_overlay_window.
class FakeOverlayOnVideoFeed:
""" Implement empty methods to replicate OverlayOnVideoFeed. """
Example usage:
video_source = 0
source_window = OverlayOnVideoFeed(video_source)
def start(self):
""" Intentionally Blank. """
duplicate_window = DuplicateOverlayWindow()
duplicate_window.set_source_window(source_window)
def stop(self):
""" Intentionally Blank. """
"""
def __init__(self):
def on_record_start(self):
""" Intentionally Blank. """
#pylint: disable=super-init-not-called
self.vtk_overlay_window = VTKOverlayWindow()
self.update_rate = 30
self.img = None
self.timer = None
def on_record_stop(self):
""" Intentionally Blank. """
def set_source_window(self, source_window):
""" Set the source window.
:param source_window: The window that contains the image to copy. """
self.source_window = source_window
def set_roi(self):
""" Intentionally Blank. """
def update(self):
""" Update the frame with a new background image."""
self.vtk_overlay_window.set_video_image(self.source_window.img)
self.vtk_overlay_window._RenderWindow.Render()
class FakeVTKOverlayWindow:
""" Implement empty methods to replicate VTKOverlayWindow. """
def save_scene_to_file(self, fname):
""" Intentionally Blank. """
class MonoViewer(QtWidgets.QWidget):
"""
Generates a VTK interactor UI with a video stream as background
:param video_source: OpenCV compatible video source (int or filename)
"""
def __init__(self, video_source):
super().__init__()
LOGGER.info("Creating Mono Viewer")
def get_foreground_camera(self):
""" Intentionally Blank. """
self.overlay_window = OverlayOnVideoFeed(video_source)
def set_foreground_camera(self, camera):
""" Intentionally Blank. """
self.UI = UI(self.overlay_window)
def set_screen(self):
""" Intentionally Blank. """
def add_vtk_models(self, models):
"""
Add vtk model overlays to the window
:param models: list of vtk models to add
:type models: list of VTKSurfaceModels
"""
self.UI.add_vtk_models(models)
def set_geometry(self):
""" Intentionally Blank. """
def start(self):
""" Start the viewer. """
self.overlay_window.start()
def add_vtk_models(self, models):
""" Intentionally Blank. """
class StereoViewerBase(QtWidgets.QWidget):
"""
Base class for StereoViewers.
"""
def setup_widgets(self):
""" Sync the cameras between the widgets,
and setup Qt output signal. """
self.UI = UI(self.ui_window)
Child classes implment the left/right/ui video feeds as appropriate.
"""
self.left_overlay = self.left_view.vtk_overlay_window
self.right_overlay = self.right_view.vtk_overlay_window
self.ui_overlay = self.ui_window.vtk_overlay_window
def __init__(self):
super().__init__()
self.sync_camera_view_between_windows()
self.primary_screen = None
self.all_screens = None
self.num_additional_screens = None
self.UI.screenshot_button.clicked.connect(self.on_screenshot_clicked)
self.UI.record_button.clicked.connect(self.on_record_start_clicked)
self.UI.crop_button.clicked.connect(self.on_crop_clicked)
self.UI.autocrop_button.clicked.connect(self.on_autocrop_started)
self.UI.exit_signal.connect(self.run_before_quit)
self.auto_cropper = AutoCropBlackBorder(threshold=5)
self.autocrop_timer = QTimer()
self.autocrop_timer.timeout.connect(self.update_autocrop)
self.sync_camera_view_between_windows()
def sync_camera_view_between_windows(self):
"""
Set all the foreground cameras to the same vtkCamera,
......@@ -106,8 +99,10 @@ class StereoViewerBase(QtWidgets.QWidget):
:type output_screens: List of ints
"""
screen_controller = ScreenController()
self.primary, self.screens = screen_controller.list_of_screens()
self.num_additional_screens = len(self.screens)
self.primary_screen, self.all_screens = \
screen_controller.list_of_screens()
self.num_additional_screens = len(self.all_screens)
if self.sufficent_displays_for_stereo_view():
self.set_widget_screens(output_screens)
......@@ -126,7 +121,8 @@ class StereoViewerBase(QtWidgets.QWidget):
:param screens_to_use: List of QScreen objects.
"""
ordered_screens = [self.primary, self.screens[0], self.screens[1]]
ordered_screens = \
[self.primary_screen, self.all_screens[0], self.all_screens[1]]
UI_screen = ordered_screens[screens_to_use[0]-1]
left_screen = ordered_screens[screens_to_use[1]-1]
......@@ -140,10 +136,11 @@ class StereoViewerBase(QtWidgets.QWidget):
self.maximize_left_and_right_widgets()
def are_widgets_on_different_screens(self, screens):
#pylint:disable=no-self-use
"""
Check if each widget has been placed on it's own screen.
:param screens: List of screen numbers corresponding to the screen
each widget is displayed on.
each widget is displayed on.
:type screens: list of ints
"""
......@@ -181,7 +178,7 @@ class StereoViewerBase(QtWidgets.QWidget):
self.left_view.start()
self.right_view.start()
self.ui_window.start()
self.ui_view.start()
def run_before_quit(self):
"""
......@@ -191,32 +188,141 @@ class StereoViewerBase(QtWidgets.QWidget):
self.left_view.stop()
self.right_view.stop()
self.ui_window.stop()
self.ui_view.stop()
def on_screenshot_clicked(self):
""" Save a screenshot to disk, using date and time as filename """
fname_base = 'outputs/' \
+ datetime.datetime.now().strftime("%Y-%m-%d.%H-%M-%S")
if self.__class__.__name__ == "MonoViewer":
fname_left = fname_base + '-MONO.png'
else:
fname_left = fname_base + '-LEFT.png'
fname_right = fname_base + '-RIGHT.png'
self.left_overlay.save_scene_to_file(fname_left)
self.right_overlay.save_scene_to_file(fname_right)
def on_record_start_clicked(self):
""" Start recording a video. In the MockStereoViewer we only need
to record the 'left' input, as the right view is a duplicate of the
left.
The proper StereoViewer class extends this method to also record
the right view."""
fname_base = 'outputs/' + \
datetime.datetime.now().strftime("%Y-%m-%d.%H-%M-%S")
if self.__class__.__name__ == "MonoViewer":
fname_left = fname_base + '-MONO.avi'
else:
fname_left = fname_base + '-LEFT.avi'
fname_right = fname_base + '-RIGHT.avi'
self.left_view.output_filename = fname_left
self.left_view.on_record_start()
self.right_view.output_filename = fname_right
self.right_view.on_record_start()
self.UI.record_button.setText("Stop recording")
self.UI.record_button.clicked.disconnect()
self.UI.record_button.clicked.connect(self.on_record_stop_clicked)
def on_record_stop_clicked(self):
""" Stop recording data to file and restore button settings. """
self.left_view.on_record_stop()
self.right_view.on_record_stop()
self.UI.record_button.setText("Record video")
self.UI.record_button.clicked.disconnect()
self.UI.record_button.clicked.connect(self.on_record_start_clicked)
def on_crop_clicked(self):
""" Set the ROI on the left view, and copy it to the right. """
self.left_view.set_roi()
self.right_view.roi = self.left_view.roi
def on_autocrop_started(self):
""" Start auto cropping. """
self.UI.autocrop_button.setText("Disable Auto Crop")
self.UI.autocrop_button.clicked.disconnect()
self.UI.autocrop_button.clicked.connect(self.on_autocrop_stopped)
self.autocrop_timer.start(500)
def on_autocrop_stopped(self):
""" Stop auto cropping. """
self.UI.autocrop_button.setText("Enable Auto Crop")
self.UI.autocrop_button.clicked.disconnect()
self.UI.autocrop_button.clicked.connect(self.on_autocrop_started)
self.autocrop_timer.stop()
self.set_all_roi(None)
def update_autocrop(self):
""" Automatically crop the incoming video stream using AutoCropper.
"""
roi = self.auto_cropper.get_roi(self.left_view.img)
self.set_all_roi(roi)
def set_all_roi(self, roi):
""" Set the roi for left/right views. """
self.left_view.roi = roi
self.right_view.roi = roi
class MonoViewer(StereoViewerBase):
"""
Generates a VTK interactor UI with a single video stream as background.
:param video_source: OpenCV compatible video source (int or filename)
Only use the left_view of StereoViewerBase. Set the other views to
non-existent views.
"""
def __init__(self, video_source):
LOGGER.info("Creating Mono Viewer")
self.left_view = OverlayOnVideoFeedCropRecord(video_source)
self.left_overlay = self.left_view.vtk_overlay_window
self.right_view = FakeOverlayOnVideoFeed()
self.right_overlay = FakeVTKOverlayWindow()
self.ui_view = FakeOverlayOnVideoFeed()
self.ui_overlay = FakeVTKOverlayWindow()
self.UI = UI(self.left_view)
super().__init__()
class MockStereoViewer(StereoViewerBase):
"""
Mock stereo viewer, duplicating a single webcam input
Mock stereo viewer, duplicating a single camera input
to multiple screens.
:param video_source: OpenCV compatible video source (int or filename)
"""
def __init__(self, video_source):
super().__init__()
LOGGER.info("Creating Mock Stereo Viewwer")
self.left_view = OverlayOnVideoFeed(video_source)
self.left_view = OverlayOnVideoFeedCropRecord(video_source)
self.left_overlay = self.left_view.vtk_overlay_window
self.right_view = DuplicateOverlayWindow()
self.right_view.set_source_window(self.left_view)