User Interface Examples

Detection User Interface

Run the Detection User Interface Example

  1. Retrieve the example, and execute it on the SoM:

curl -LJO https://github.com/varigit/pyvar/raw/master/examples/ml/detection/ui_detection.py

python3 ui_detection.py

  1. The output should be similar as the one below:

User Interface Example

ui_objects

User Interface Options

ui_options

User Interface Detected

ui_objects_detected
User Interface Detection Example Source code: ui_detection.py
  1# Copyright 2021 Variscite LTD
  2# SPDX-License-Identifier: BSD-3-Clause
  3
  4"""
  5This script performs real-time object detection using the TFLiteInterpreter
  6engine and displays the results using a GTK GUI.
  7
  8It performs the following steps:
  9
 101. Retrieves the detection package using a HTTPS retriever instance.
 112. Creates a GTK GUI window to display the detected objects.
 123. Loads the labels from the label file.
 134. Creates a TFLiteInterpreter engine instance, a Resizer instance, and an
 14   Overlay instance.
 155. Resizes each frame of the input video to the engine's input size.
 166. Runs inference and gets the output for each frame.
 177. Creates a list of detected objects based on the output of the inference and
 18   the user-selected objects to be detected.
 198. Draws the output image with the detected objects and other information using
 20   the Overlay instance.
 219. Displays the output video in the GTK GUI window.
 22
 23Example:
 24
 25To run this script:
 26    $ python3 ui_detection.py
 27
 28Args:
 29None.
 30
 31Returns:
 32None.
 33"""
 34
 35import cv2
 36import numpy as np
 37import threading
 38
 39import gi
 40gi.require_versions({'GdkPixbuf': "2.0", 'Gtk': "3.0"})
 41from gi.repository.GdkPixbuf import Colorspace, Pixbuf
 42from gi.repository import GLib, Gtk
 43
 44from pyvar.ml.engines.tflite import TFLiteInterpreter
 45from pyvar.ml.utils.framerate import Framerate
 46from pyvar.ml.utils.label import Label
 47from pyvar.ml.utils.overlay import Overlay
 48from pyvar.ml.utils.resizer import Resizer
 49from pyvar.ml.utils.retriever_https import HTTPS
 50from pyvar.multimedia.helper import Multimedia
 51
 52SSD_LABELS_LIST = [
 53    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
 54    "truck", "boat", "traffic light", "fire hydrant", "stop sign",
 55    "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
 56    "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag",
 57    "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
 58    "baseball bat", "baseball glove", "skateboard", "surfboard",
 59    "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon",
 60    "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot",
 61    "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant",
 62    "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote",
 63    "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
 64    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
 65    "hair drier", "toothbrush"]
 66
 67
 68class ObjectSelection(Gtk.Frame):
 69    def __init__(self, parent, exclude_list):
 70        super().__init__()
 71        self.parent = parent
 72        self.exclude_list = exclude_list
 73        labels_list = self.exclude_list.copy()
 74        labels_list.sort()
 75
 76        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
 77        self.add(vertical_box)
 78
 79        scrolled_window = Gtk.ScrolledWindow()
 80
 81        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
 82        quit_button = Gtk.Button.new_with_label('Quit')
 83        quit_button.connect('clicked', self.on_quit_button_clicked)
 84        back_button = Gtk.Button.new_with_label('Back')
 85        back_button.connect('clicked', self.on_back_button_clicked)
 86        horizontal_box.pack_start(back_button, True, True, 10)
 87        horizontal_box.pack_start(quit_button, True, True, 10)
 88        vertical_box.pack_start(horizontal_box, False, True, 10)
 89        vertical_box.pack_start(scrolled_window, True, True, 10)
 90
 91        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
 92        scrolled_window.add(vertical_box)
 93
 94        for label in labels_list:
 95            horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
 96            switch_button = Gtk.Switch()
 97            switch_button.set_active(False)
 98            switch_button.connect('notify::active',
 99                                  self.on_object_switch_activated, label)
100            label_name = Gtk.Label.new(label)
101            horizontal_box.pack_start(label_name, True, True, 100)
102            horizontal_box.pack_start(switch_button, False, True, 100)
103            vertical_box.pack_start(horizontal_box, True, True, 10)
104
105    def on_quit_button_clicked(self, button):
106        Gtk.main_quit()
107
108    def on_back_button_clicked(self, button):
109        self.parent.set_current_page(0)
110
111    def on_object_switch_activated(self, switch, gparam, obj):
112        if switch.get_active():
113            if obj in self.exclude_list:
114                self.exclude_list.remove(obj)
115        else:
116            if obj not in self.exclude_list:
117                self.exclude_list.append(obj)
118
119
120class RealTimeDetection(Gtk.Frame):
121    def __init__(self, parent, exclude_list):
122        super().__init__()
123        self.parent = parent
124        self.exclude_list = exclude_list
125        self.model_file_path = None
126        self.label_file_path = None
127
128        https = HTTPS()
129
130        if https.retrieve_package(category="detection"):
131            self.model_file_path = https.model
132            self.label_file_path = https.label
133            
134        labels = Label(self.label_file_path)
135        labels.read_labels("detection")
136 
137        self.labels = labels.list
138
139        self.engine = None
140        self.interpreter = None
141        self.input_details = None
142        self.output_details = None
143        self.pixbuf = None
144
145        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
146        self.add(vertical_box)
147
148        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
149        quit_button = Gtk.Button.new_with_label('Quit')
150        quit_button.connect('clicked', self.on_quit_button_clicked)
151        objects_button = Gtk.Button.new_with_label('Objects')
152        objects_button.connect('clicked', self.on_objects_button_clicked)
153        horizontal_box.pack_start(objects_button, True, True, 10)
154        horizontal_box.pack_start(quit_button, True, True, 10)
155        vertical_box.pack_start(horizontal_box, True, False, 10)
156
157        self.displayed_image = Gtk.Image()
158        image_box = Gtk.Box(spacing=5)
159        image_box.pack_start(self.displayed_image, True, True, 0)
160        vertical_box.pack_start(image_box, True, True, 5)
161
162        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
163
164        inference_time_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
165        inference_label = Gtk.Label()
166        inference_label.set_markup('INFERENCE TIME:')
167        self.inference_value_label = Gtk.Label.new(None)
168        inference_time_box.pack_start(inference_label, False, True, 10)
169        inference_time_box.pack_start(self.inference_value_label, False, False, 10)
170
171        fps_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
172        fps_label = Gtk.Label()
173        fps_label.set_markup('FPS:')
174        self.fps_value_label = Gtk.Label.new(None)
175        fps_box.pack_start(fps_label, False, True, 10)
176        fps_box.pack_start(self.fps_value_label, False, False, 10)
177
178        horizontal_box.pack_start(inference_time_box, True, True, 10)
179        horizontal_box.pack_start(fps_box, True, True, 10)
180        vertical_box.pack_start(horizontal_box, True, False, 10)
181
182        self.start_interpreter()
183        self.run_application()
184
185    def on_quit_button_clicked(self, button):
186        Gtk.main_quit()
187
188    def on_objects_button_clicked(self, button):
189        self.parent.set_current_page(1)
190
191    def set_displayed_image(self, image):
192        image = cv2.resize(image, (420, 340))
193        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
194        height, width = image.shape[:2]
195        arr = np.ndarray.tobytes(image)
196        self.pixbuf = Pixbuf.new_from_data(
197                             arr, Colorspace.RGB, False, 8,
198                             width, height, width*3, None, None)
199        self.displayed_image.set_from_pixbuf(self.pixbuf)
200        self.pixbuf = None
201
202    def run_application(self):
203        thread = threading.Thread(target=self.image_detection)
204        thread.daemon = True
205        thread.start()
206
207    def start_interpreter(self):
208        self.engine = TFLiteInterpreter(self.model_file_path)
209
210    def image_detection(self):
211        resizer = Resizer()
212        resizer.set_sizes(engine_input_details=self.engine.input_details)
213
214        camera = Multimedia("/dev/video1", resolution="vga")
215        camera.set_v4l2_config()
216        
217        framerate = Framerate()
218
219        draw = Overlay()
220        draw.inference_time_info = False
221        draw.scores_info = True
222        draw.extra_info = False
223        draw.framerate_info = False
224        
225        while camera.loop:
226            with framerate.fpsit():
227                frame = camera.get_frame()
228                resizer.resize_frame(frame)
229
230                self.engine.set_input(resizer.frame_resized)
231                self.engine.run_inference()
232
233                positions = self.engine.get_output(0, squeeze=True)
234                classes = self.engine.get_output(1, squeeze=True)
235                scores = self.engine.get_output(2, squeeze=True)
236                
237                result = []
238                for idx, score in enumerate(scores):
239                    if score > 0.5 and (self.labels[classes[idx]] not in self.exclude_list):
240                        result.append({'pos': positions[idx], '_id': classes[idx]})
241                
242                output_frame = draw.info(category="detection",
243                                         image=resizer.frame,
244                                         top_result=result,
245                                         labels=self.labels,
246                                         inference_time=None,
247                                         model_name=None,
248                                         source_file=camera.dev.name,
249                                         fps=None)
250
251            GLib.idle_add(self.inference_value_label.set_text,
252                          f"{self.engine.inference_time}")
253            GLib.idle_add(self.fps_value_label.set_text,
254                          f"{int(framerate.fps)}")
255            GLib.idle_add(self.set_displayed_image, output_frame)
256
257
258class UserInterfaceDetectionExample(Gtk.Window):
259    def __init__(self):
260        super().__init__(title='User Interface Detection Example')
261        self.fullscreen()
262        exclude_list = SSD_LABELS_LIST.copy()
263
264        container = Gtk.Notebook()
265        container.set_show_tabs(False)
266        self.add(container)
267
268        realtime_page = RealTimeDetection(container, exclude_list)
269        container.append_page(realtime_page)
270
271        label_selection_page = ObjectSelection(container, exclude_list)
272        container.append_page(label_selection_page)
273
274
275if __name__ == "__main__":
276    app = UserInterfaceDetectionExample()
277    app.connect('delete-event', Gtk.main_quit)
278    app.show_all()
279    Gtk.main()