User Interface Examples

Detection User Interface

Run the Detection User Interface Example

  1. Retrieve the example, and execute it on the SoM:

curl -LJO https://github.com/varigit/pyvar/raw/master/examples/ml/detection/ui_detection.py

python3 ui_detection.py

  1. The output should be similar as the one below:

User Interface Example

ui_objects

User Interface Options

ui_options

User Interface Detected

ui_objects_detected
User Interface Detection Example Source code: ui_detection.py
  1# Copyright 2021 Variscite LTD
  2# SPDX-License-Identifier: BSD-3-Clause
  3
  4import cv2
  5import numpy as np
  6import threading
  7
  8import gi
  9gi.require_versions({'GdkPixbuf': "2.0", 'Gtk': "3.0"})
 10from gi.repository.GdkPixbuf import Colorspace, Pixbuf
 11from gi.repository import GLib, Gtk
 12
 13from pyvar.ml.engines.tflite import TFLiteInterpreter
 14from pyvar.ml.utils.framerate import Framerate
 15from pyvar.ml.utils.label import Label
 16from pyvar.ml.utils.overlay import Overlay
 17from pyvar.ml.utils.resizer import Resizer
 18from pyvar.ml.utils.retriever import FTP
 19from pyvar.multimedia.helper import Multimedia
 20
 21SSD_LABELS_LIST = [
 22    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
 23    "truck", "boat", "traffic light", "fire hydrant", "stop sign",
 24    "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
 25    "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag",
 26    "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
 27    "baseball bat", "baseball glove", "skateboard", "surfboard",
 28    "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon",
 29    "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot",
 30    "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant",
 31    "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote",
 32    "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
 33    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
 34    "hair drier", "toothbrush"]
 35
 36
 37class ObjectSelection(Gtk.Frame):
 38    def __init__(self, parent, exclude_list):
 39        super().__init__()
 40        self.parent = parent
 41        self.exclude_list = exclude_list
 42        labels_list = self.exclude_list.copy()
 43        labels_list.sort()
 44
 45        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
 46        self.add(vertical_box)
 47
 48        scrolled_window = Gtk.ScrolledWindow()
 49
 50        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
 51        quit_button = Gtk.Button.new_with_label('Quit')
 52        quit_button.connect('clicked', self.on_quit_button_clicked)
 53        back_button = Gtk.Button.new_with_label('Back')
 54        back_button.connect('clicked', self.on_back_button_clicked)
 55        horizontal_box.pack_start(back_button, True, True, 10)
 56        horizontal_box.pack_start(quit_button, True, True, 10)
 57        vertical_box.pack_start(horizontal_box, False, True, 10)
 58        vertical_box.pack_start(scrolled_window, True, True, 10)
 59
 60        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
 61        scrolled_window.add(vertical_box)
 62
 63        for label in labels_list:
 64            horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
 65            switch_button = Gtk.Switch()
 66            switch_button.set_active(False)
 67            switch_button.connect('notify::active',
 68                                  self.on_object_switch_activated, label)
 69            label_name = Gtk.Label.new(label)
 70            horizontal_box.pack_start(label_name, True, True, 100)
 71            horizontal_box.pack_start(switch_button, False, True, 100)
 72            vertical_box.pack_start(horizontal_box, True, True, 10)
 73
 74    def on_quit_button_clicked(self, button):
 75        Gtk.main_quit()
 76
 77    def on_back_button_clicked(self, button):
 78        self.parent.set_current_page(0)
 79
 80    def on_object_switch_activated(self, switch, gparam, obj):
 81        if switch.get_active():
 82            if obj in self.exclude_list:
 83                self.exclude_list.remove(obj)
 84        else:
 85            if obj not in self.exclude_list:
 86                self.exclude_list.append(obj)
 87
 88
 89class RealTimeDetection(Gtk.Frame):
 90    def __init__(self, parent, exclude_list):
 91        super().__init__()
 92        self.parent = parent
 93        self.exclude_list = exclude_list
 94        self.model_file_path = None
 95        self.label_file_path = None
 96
 97        ftp = FTP()
 98
 99        if ftp.retrieve_package(category="detection"):
100            self.model_file_path = ftp.model
101            self.label_file_path = ftp.label
102            
103        labels = Label(self.label_file_path)
104        labels.read_labels("detection")
105 
106        self.labels = labels.list
107
108        self.engine = None
109        self.interpreter = None
110        self.input_details = None
111        self.output_details = None
112        self.pixbuf = None
113
114        vertical_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
115        self.add(vertical_box)
116
117        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
118        quit_button = Gtk.Button.new_with_label('Quit')
119        quit_button.connect('clicked', self.on_quit_button_clicked)
120        objects_button = Gtk.Button.new_with_label('Objects')
121        objects_button.connect('clicked', self.on_objects_button_clicked)
122        horizontal_box.pack_start(objects_button, True, True, 10)
123        horizontal_box.pack_start(quit_button, True, True, 10)
124        vertical_box.pack_start(horizontal_box, True, False, 10)
125
126        self.displayed_image = Gtk.Image()
127        image_box = Gtk.Box(spacing=5)
128        image_box.pack_start(self.displayed_image, True, True, 0)
129        vertical_box.pack_start(image_box, True, True, 5)
130
131        horizontal_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
132
133        inference_time_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
134        inference_label = Gtk.Label()
135        inference_label.set_markup('INFERENCE TIME:')
136        self.inference_value_label = Gtk.Label.new(None)
137        inference_time_box.pack_start(inference_label, False, True, 10)
138        inference_time_box.pack_start(self.inference_value_label, False, False, 10)
139
140        fps_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
141        fps_label = Gtk.Label()
142        fps_label.set_markup('FPS:')
143        self.fps_value_label = Gtk.Label.new(None)
144        fps_box.pack_start(fps_label, False, True, 10)
145        fps_box.pack_start(self.fps_value_label, False, False, 10)
146
147        horizontal_box.pack_start(inference_time_box, True, True, 10)
148        horizontal_box.pack_start(fps_box, True, True, 10)
149        vertical_box.pack_start(horizontal_box, True, False, 10)
150
151        self.start_interpreter()
152        self.run_application()
153
154    def on_quit_button_clicked(self, button):
155        Gtk.main_quit()
156
157    def on_objects_button_clicked(self, button):
158        self.parent.set_current_page(1)
159
160    def set_displayed_image(self, image):
161        image = cv2.resize(image, (420, 340))
162        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
163        height, width = image.shape[:2]
164        arr = np.ndarray.tobytes(image)
165        self.pixbuf = Pixbuf.new_from_data(
166                             arr, Colorspace.RGB, False, 8,
167                             width, height, width*3, None, None)
168        self.displayed_image.set_from_pixbuf(self.pixbuf)
169        self.pixbuf = None
170
171    def run_application(self):
172        thread = threading.Thread(target=self.image_detection)
173        thread.daemon = True
174        thread.start()
175
176    def start_interpreter(self):
177        self.engine = TFLiteInterpreter(self.model_file_path)
178
179    def image_detection(self):
180        resizer = Resizer()
181        resizer.set_sizes(engine_input_details=self.engine.input_details)
182
183        camera = Multimedia("/dev/video1", resolution="vga")
184        camera.set_v4l2_config()
185        
186        framerate = Framerate()
187
188        draw = Overlay()
189        draw.inference_time_info = False
190        draw.scores_info = True
191        draw.extra_info = False
192        draw.framerate_info = False
193        
194        while camera.loop:
195            with framerate.fpsit():
196                frame = camera.get_frame()
197                resizer.resize_frame(frame)
198
199                self.engine.set_input(resizer.frame_resized)
200                self.engine.run_inference()
201
202                positions = self.engine.get_output(0, squeeze=True)
203                classes = self.engine.get_output(1, squeeze=True)
204                scores = self.engine.get_output(2, squeeze=True)
205                
206                result = []
207                for idx, score in enumerate(scores):
208                    if score > 0.5 and (self.labels[classes[idx]] not in self.exclude_list):
209                        result.append({'pos': positions[idx], '_id': classes[idx]})
210                
211                output_frame = draw.info(category="detection",
212                                         image=resizer.frame,
213                                         top_result=result,
214                                         labels=self.labels,
215                                         inference_time=None,
216                                         model_name=None,
217                                         source_file=camera.dev.name,
218                                         fps=None)
219
220            GLib.idle_add(self.inference_value_label.set_text,
221                          f"{self.engine.inference_time}")
222            GLib.idle_add(self.fps_value_label.set_text,
223                          f"{int(framerate.fps)}")
224            GLib.idle_add(self.set_displayed_image, output_frame)
225
226
227class UserInterfaceDetectionExample(Gtk.Window):
228    def __init__(self):
229        super().__init__(title='User Interface Detection Example')
230        self.fullscreen()
231        exclude_list = SSD_LABELS_LIST.copy()
232
233        container = Gtk.Notebook()
234        container.set_show_tabs(False)
235        self.add(container)
236
237        realtime_page = RealTimeDetection(container, exclude_list)
238        container.append_page(realtime_page)
239
240        label_selection_page = ObjectSelection(container, exclude_list)
241        container.append_page(label_selection_page)
242
243
244if __name__ == "__main__":
245    app = UserInterfaceDetectionExample()
246    app.connect('delete-event', Gtk.main_quit)
247    app.show_all()
248    Gtk.main()