Dynamic engine options in UI for blender / ffmpeg (#66)

* Make sure progress UI updates occur on main thread

* Cleanup unnecessary code in FFMPEG

* Cleanup extension matching

* Make sure supported_extensions is now called as a method everywhere

* Fix add_job crashing

* Update the renderer to reflect the current file type

* Sort engine versions from newest to oldest

* Consolidate Project Group and Server Group

* Split UI options into its own file for easier updating

* Add ffmpeg ui stem

src/api/api_server.py

@@ -423,10 +423,12 @@ def renderer_info():
         # Get all installed versions of engine
         installed_versions = EngineManager.all_versions_for_engine(engine.name())
         if installed_versions:
-            install_path = installed_versions[0]['path']
+            # fixme: using system versions only because downloaded versions may have permissions issues
+            system_installed_versions = [x for x in installed_versions if x['type'] == 'system']
+            install_path = system_installed_versions[0]['path'] if system_installed_versions else installed_versions[0]['path']
             renderer_data[engine.name()] = {'is_available': RenderQueue.is_available_for_job(engine.name()),
                                             'versions': installed_versions,
-                                            'supported_extensions': engine.supported_extensions,
+                                            'supported_extensions': engine.supported_extensions(),
                                             'supported_export_formats': engine(install_path).get_output_formats()}
     return renderer_data
 

src/engines/blender/blender_engine.py

@@ -10,7 +10,6 @@ logger = logging.getLogger()
 class Blender(BaseRenderEngine):
 
     install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender']
-    supported_extensions = ['.blend']
     binary_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender'}
 
     @staticmethod
@@ -23,6 +22,14 @@ class Blender(BaseRenderEngine):
         from src.engines.blender.blender_worker import BlenderRenderWorker
         return BlenderRenderWorker
 
+    def ui_options(self):
+        from src.engines.blender.blender_ui import BlenderUI
+        return BlenderUI.get_options(self)
+
+    @staticmethod
+    def supported_extensions():
+        return ['blend']
+
     def version(self):
         version = None
         try:
@@ -150,13 +157,6 @@ class Blender(BaseRenderEngine):
         render_engines = [x.strip() for x in engine_output.split('Blender Engine Listing:')[-1].strip().splitlines()]
         return render_engines
 
-    # UI and setup
-    def get_options(self):
-        options = [
-            {'name': 'engine', 'options': self.supported_render_engines()},
-        ]
-        return options
-
     def perform_presubmission_tasks(self, project_path):
         packed_path = self.pack_project_file(project_path, timeout=30)
         return packed_path

src/engines/blender/blender_ui.py

@@ -0,0 +1,8 @@
+
+class BlenderUI:
+    @staticmethod
+    def get_options(instance):
+        options = [
+            {'name': 'engine', 'options': instance.supported_render_engines()},
+        ]
+        return options

src/engines/core/base_engine.py

@@ -47,7 +47,10 @@ class BaseRenderEngine(object):
     def worker_class():  # override when subclassing to link worker class
         raise NotImplementedError("Worker class not implemented")
 
-    def get_help(self):
+    def ui_options(self):  # override to return options for ui
+        return {}
+
+    def get_help(self):  # override if renderer uses different help flag
         path = self.renderer_path()
         if not path:
             raise FileNotFoundError("renderer path not found")
@@ -56,7 +59,7 @@ class BaseRenderEngine(object):
         return help_doc
 
     def get_project_info(self, project_path, timeout=10):
-        raise NotImplementedError(f"get_project_info not implemented for {cls.__name__}")
+        raise NotImplementedError(f"get_project_info not implemented for {self.__name__}")
 
     @classmethod
     def get_output_formats(cls):

src/engines/core/base_worker.py

@@ -47,7 +47,7 @@ class BaseRenderWorker(Base):
                  name=None):
 
         if not ignore_extensions:
-            if not any(ext in input_path for ext in self.engine.supported_extensions):
+            if not any(ext in input_path for ext in self.engine.supported_extensions()):
                 err_meg = f'Cannot find valid project with supported file extension for {self.engine.name()} renderer'
                 logger.error(err_meg)
                 raise ValueError(err_meg)

src/engines/engine_manager.py

@@ -76,7 +76,9 @@ class EngineManager:
 
     @classmethod
     def all_versions_for_engine(cls, engine):
-        return [x for x in cls.all_engines() if x['engine'] == engine]
+        versions = [x for x in cls.all_engines() if x['engine'] == engine]
+        sorted_versions = sorted(versions, key=lambda x: x['version'], reverse=True)
+        return sorted_versions
 
     @classmethod
     def newest_engine_version(cls, engine, system_os=None, cpu=None):
@@ -84,9 +86,8 @@ class EngineManager:
         cpu = cpu or current_system_cpu()
 
         try:
-            filtered = [x for x in cls.all_engines() if x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu]
-            versions = sorted(filtered, key=lambda x: x['version'], reverse=True)
-            return versions[0]
+            filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and x['cpu'] == cpu]
+            return filtered[0]
         except IndexError:
             logger.error(f"Cannot find newest engine version for {engine}-{system_os}-{cpu}")
             return None
@@ -234,10 +235,11 @@ class EngineManager:
     @classmethod
     def engine_for_project_path(cls, path):
         name, extension = os.path.splitext(path)
+        extension = extension.lower().strip('.')
         for engine in cls.supported_engines():
-            if extension in engine.supported_extensions:
+            if extension in engine.supported_extensions():
                 return engine
-        undefined_renderer_support = [x for x in cls.supported_engines() if not x.supported_extensions]
+        undefined_renderer_support = [x for x in cls.supported_engines() if not x.supported_extensions()]
         return undefined_renderer_support[0]
 
 

src/engines/ffmpeg/ffmpeg_engine.py

@@ -18,6 +18,20 @@ class FFMPEG(BaseRenderEngine):
         from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker
         return FFMPEGRenderWorker
 
+    def ui_options(self):
+        from src.engines.ffmpeg.ffmpeg_ui import FFMPEGUI
+        return FFMPEGUI.get_options(self)
+
+    @classmethod
+    def supported_extensions(cls):
+        help_text = (subprocess.check_output([cls().renderer_path(), '-h', 'full'], stderr=subprocess.STDOUT)
+                     .decode('utf-8'))
+        found = re.findall('extensions that .* is allowed to access \(default "(.*)"', help_text)
+        found_extensions = set()
+        for match in found:
+            found_extensions.update(match.split(','))
+        return list(found_extensions)
+
     def version(self):
         version = None
         try:
@@ -31,15 +45,11 @@ class FFMPEG(BaseRenderEngine):
         return version
 
     def get_project_info(self, project_path, timeout=10):
-        return self.get_video_info_ffprobe(project_path)
-
-    @staticmethod
-    def get_video_info_ffprobe(video_path):
         try:
             # Run ffprobe and parse the output as JSON
             cmd = [
                 'ffprobe', '-v', 'quiet', '-print_format', 'json',
-                '-show_streams', '-select_streams', 'v', video_path
+                '-show_streams', '-select_streams', 'v', project_path
             ]
             result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
             video_info = json.loads(result.stdout)
@@ -85,7 +95,7 @@ class FFMPEG(BaseRenderEngine):
         try:
             formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
                                                  timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
-            pattern = '(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)\r'
+            pattern = '(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)'
             all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
             return all_formats
         except Exception as e:
@@ -105,7 +115,7 @@ class FFMPEG(BaseRenderEngine):
         return found_extensions
 
     def get_output_formats(self):
-        return [x for x in self.get_all_formats() if 'E' in x['type'].upper()]
+        return [x['id'] for x in self.get_all_formats() if 'E' in x['type'].upper()]
 
     def get_frame_count(self, path_to_file):
         raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
@@ -117,7 +127,8 @@ class FFMPEG(BaseRenderEngine):
             return frame_number
 
     def get_arguments(self):
-        help_text = subprocess.check_output([self.renderer_path(), '-h', 'long'], stderr=subprocess.STDOUT).decode('utf-8')
+        help_text = (subprocess.check_output([self.renderer_path(), '-h', 'long'], stderr=subprocess.STDOUT)
+                     .decode('utf-8'))
         lines = help_text.splitlines()
 
         options = {}

src/engines/ffmpeg/ffmpeg_ui.py

@@ -0,0 +1,5 @@
+class FFMPEGUI:
+    @staticmethod
+    def get_options(instance):
+        options = []
+        return options

src/engines/ffmpeg/ffmpeg_worker.py

@@ -10,15 +10,9 @@ class FFMPEGRenderWorker(BaseRenderWorker):
 
     engine = FFMPEG
 
-    def __init__(self, input_path, output_path, args=None, parent=None, name=None):
-        super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args,
-                                                 parent=parent, name=name)
-
-        stream_info = subprocess.check_output([self.renderer_path, "-i",  # https://stackoverflow.com/a/61604105
-                                               input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y",
-                                                     "/dev/null"], stderr=subprocess.STDOUT).decode('utf-8')
-        found_frames = re.findall('frame=\s*(\d+)', stream_info)
-        self.project_length = found_frames[-1] if found_frames else '-1'
+    def __init__(self, input_path, output_path, engine_path, args=None, parent=None, name=None):
+        super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path,
+                                                 engine_path=engine_path, args=args, parent=parent, name=name)
         self.current_frame = -1
 
     def generate_worker_subprocess(self):

src/ui/add_job.py

@@ -5,7 +5,7 @@ import socket
 import threading
 
 import psutil
-from PyQt6.QtCore import QThread, pyqtSignal, Qt
+from PyQt6.QtCore import QThread, pyqtSignal, Qt, pyqtSlot
 from PyQt6.QtWidgets import (
     QApplication, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QPushButton, QFileDialog, QSpinBox, QComboBox,
     QGroupBox, QCheckBox, QProgressBar, QPlainTextEdit, QDoubleSpinBox, QMessageBox, QListWidget, QListWidgetItem
@@ -21,10 +21,11 @@ from src.utilities.zeroconf_server import ZeroconfServer
 class NewRenderJobForm(QWidget):
     def __init__(self, project_path=None):
         super().__init__()
-
         self.project_path = project_path
 
         # UI
+        self.project_group = None
+        self.load_file_group = None
         self.current_engine_options = None
         self.file_format_combo = None
         self.renderer_options_layout = None
@@ -73,41 +74,41 @@ class NewRenderJobForm(QWidget):
         # Main Layout
         main_layout = QVBoxLayout(self)
 
-        # Scene File Group
-        scene_file_group = QGroupBox("Project")
-        scene_file_layout = QVBoxLayout(scene_file_group)
-        scene_file_picker_layout = QHBoxLayout()
-        self.scene_file_input = QLineEdit()
-        self.scene_file_input.setText(self.project_path)
-        self.scene_file_browse_button = QPushButton("Browse...")
-        self.scene_file_browse_button.clicked.connect(self.browse_scene_file)
-        scene_file_picker_layout.addWidget(self.scene_file_input)
-        scene_file_picker_layout.addWidget(self.scene_file_browse_button)
-        scene_file_layout.addLayout(scene_file_picker_layout)
+        # Loading File Group
+        self.load_file_group = QGroupBox("Loading")
+        load_file_layout = QVBoxLayout(self.load_file_group)
         # progress bar
         progress_layout = QHBoxLayout()
         self.process_progress_bar = QProgressBar()
         self.process_progress_bar.setMinimum(0)
         self.process_progress_bar.setMaximum(0)
-        self.process_progress_bar.setHidden(True)
         self.process_label = QLabel("Processing")
-        self.process_label.setHidden(True)
         progress_layout.addWidget(self.process_label)
         progress_layout.addWidget(self.process_progress_bar)
-        scene_file_layout.addLayout(progress_layout)
-        main_layout.addWidget(scene_file_group)
+        load_file_layout.addLayout(progress_layout)
+        main_layout.addWidget(self.load_file_group)
 
-        # Server Group
+        # Project Group
+        self.project_group = QGroupBox("Project")
+        server_layout = QVBoxLayout(self.project_group)
+        # File Path
+        scene_file_picker_layout = QHBoxLayout()
+        self.scene_file_input = QLineEdit()
+        self.scene_file_input.setText(self.project_path)
+        self.scene_file_browse_button = QPushButton("Browse...")
+        self.scene_file_browse_button.clicked.connect(self.browse_scene_file)
+        scene_file_picker_layout.addWidget(QLabel("File:"))
+        scene_file_picker_layout.addWidget(self.scene_file_input)
+        scene_file_picker_layout.addWidget(self.scene_file_browse_button)
+        server_layout.addLayout(scene_file_picker_layout)
         # Server List
-        self.server_group = QGroupBox("Server")
-        server_layout = QVBoxLayout(self.server_group)
         server_list_layout = QHBoxLayout()
         server_list_layout.setSpacing(0)
         self.server_input = QComboBox()
         server_list_layout.addWidget(QLabel("Hostname:"), 1)
         server_list_layout.addWidget(self.server_input, 3)
         server_layout.addLayout(server_list_layout)
-        main_layout.addWidget(self.server_group)
+        main_layout.addWidget(self.project_group)
         self.update_server_list()
         # Priority
         priority_layout = QHBoxLayout()
@@ -234,8 +235,13 @@ class NewRenderJobForm(QWidget):
         self.toggle_renderer_enablement(False)
 
     def update_renderer_info(self):
+        # get the renderer info and add them all to the ui
         self.renderer_info = self.server_proxy.get_renderer_info()
         self.renderer_type.addItems(self.renderer_info.keys())
+        # select the best renderer for the file type
+        engine = EngineManager.engine_for_project_path(self.project_path)
+        self.renderer_type.setCurrentText(engine.name().lower())
+        # refresh ui
         self.renderer_changed()
 
     def renderer_changed(self):
@@ -294,15 +300,18 @@ class NewRenderJobForm(QWidget):
             # Set the best renderer we can find
             input_path = self.scene_file_input.text()
             engine = EngineManager.engine_for_project_path(input_path)
-            index = self.renderer_type.findText(engine.name().lower())
-            if index >= 0:
-                self.renderer_type.setCurrentIndex(index)
+
+            engine_index = self.renderer_type.findText(engine.name().lower())
+            if engine_index >= 0:
+                self.renderer_type.setCurrentIndex(engine_index)
+            else:
+                self.renderer_type.setCurrentIndex(0)  #todo: find out why we don't have renderer info yet
+                # not ideal but if we don't have the renderer info we have to pick something
 
             self.output_path_input.setText(os.path.basename(input_path))
 
             # cleanup progress UI
-            self.process_progress_bar.setHidden(True)
-            self.process_label.setHidden(True)
+            self.load_file_group.setHidden(True)
             self.toggle_renderer_enablement(True)
 
             # Load scene data
@@ -331,12 +340,9 @@ class NewRenderJobForm(QWidget):
                 self.cameras_group.setHidden(True)
 
             # Dynamic Engine Options
-            engine_name = self.renderer_type.currentText()
-            engine = EngineManager.engine_with_name(engine_name)
-            # clear old options
-            clear_layout(self.renderer_options_layout)
+            clear_layout(self.renderer_options_layout)  # clear old options
             # dynamically populate option list
-            self.current_engine_options = engine().get_options()
+            self.current_engine_options = engine().ui_options()
             for option in self.current_engine_options:
                 h_layout = QHBoxLayout()
                 label = QLabel(option['name'].capitalize() + ':')
@@ -355,7 +361,7 @@ class NewRenderJobForm(QWidget):
 
     def toggle_renderer_enablement(self, enabled=False):
         """Toggle on/off all the render settings"""
-        self.server_group.setHidden(not enabled)
+        self.project_group.setHidden(not enabled)
         self.output_settings_group.setHidden(not enabled)
         self.renderer_group.setHidden(not enabled)
         self.notes_group.setHidden(not enabled)
@@ -402,14 +408,23 @@ class NewRenderJobForm(QWidget):
 
         # submit job in background thread
         self.worker_thread = SubmitWorker(window=self)
+        self.worker_thread.update_ui_signal.connect(self.update_submit_progress)
         self.worker_thread.message_signal.connect(self.after_job_submission)
         self.worker_thread.start()
 
+    @pyqtSlot(str, str)
+    def update_submit_progress(self, hostname, percent):
+        # Update the UI here. This slot will be executed in the main thread
+        self.submit_progress_label.setText(f"Transferring to {hostname} - {percent}%")
+        self.submit_progress.setMaximum(100)
+        self.submit_progress.setValue(int(percent))
+
 
 class SubmitWorker(QThread):
     """Worker class called to submit all the jobs to the server and update the UI accordingly"""
 
     message_signal = pyqtSignal(Response)
+    update_ui_signal = pyqtSignal(str, str)
 
     def __init__(self, window):
         super().__init__()
@@ -421,10 +436,7 @@ class SubmitWorker(QThread):
 
             def callback(monitor):
                 percent = f"{monitor.bytes_read / encoder_len * 100:.0f}"
-                self.window.submit_progress_label.setText(f"Transferring to {hostname} - {percent}%")
-                self.window.submit_progress.setMaximum(100)
-                self.window.submit_progress.setValue(int(percent))
-
+                self.update_ui_signal.emit(hostname, percent)
             return callback
 
         hostname = self.window.server_input.currentText()