More backend refactoring / improvements

This commit is contained in:
2026-01-07 11:27:25 -06:00
parent f6ee57fb55
commit d2a30a887f
8 changed files with 166 additions and 89 deletions

View File

@@ -380,6 +380,16 @@ def delete_job(job_id):
# Engine Info and Management:
# --------------------------------------------
@server.get('/api/installed_engines')
def get_installed_engines():
result = {}
for engine_class in EngineManager.supported_engines():
data = EngineManager.all_version_data_for_engine(engine_class.name())
if data:
result[engine_class.name()] = data
return result
@server.get('/api/engine_info')
def engine_info():
response_type = request.args.get('response_type', 'standard')
@@ -434,6 +444,39 @@ def engine_info():
return engine_data
@server.get('/api/<engine_name>/info')
def get_engine_info(engine_name):
try:
response_type = request.args.get('response_type', 'standard')
# Get all installed versions of the engine
installed_versions = EngineManager.all_version_data_for_engine(engine_name)
if not installed_versions:
return {}
result = { 'is_available': RenderQueue.is_available_for_job(engine_name),
'versions': installed_versions
}
if response_type == 'full':
with concurrent.futures.ThreadPoolExecutor() as executor:
engine_class = EngineManager.engine_class_with_name(engine_name)
en = EngineManager.get_latest_engine_instance(engine_class)
future_results = {
'supported_extensions': executor.submit(en.supported_extensions),
'supported_export_formats': executor.submit(en.get_output_formats),
'system_info': executor.submit(en.system_info)
}
for key, future in future_results.items():
result[key] = future.result()
return result
except Exception as e:
logger.error(f"Error fetching details for engine '{engine_name}': {e}")
return {}
@server.get('/api/<engine_name>/is_available')
def is_engine_available(engine_name):
return {'engine': engine_name, 'available': RenderQueue.is_available_for_job(engine_name),
@@ -442,6 +485,27 @@ def is_engine_available(engine_name):
'hostname': server.config['HOSTNAME']}
@server.get('/api/engine/<engine_name>/args')
def get_engine_args(engine_name):
try:
engine_class = EngineManager.engine_class_with_name(engine_name)
return engine_class().get_arguments()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
@server.get('/api/engine/<engine_name>/help')
def get_engine_help(engine_name):
try:
engine_class = EngineManager.engine_class_with_name(engine_name)
return engine_class().get_help()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
# --------------------------------------------
# Engine Downloads and Updates:
# --------------------------------------------
@server.get('/api/is_engine_available_to_download')
def is_engine_available_to_download():
available_result = EngineManager.version_is_available_to_download(request.args.get('engine'),
@@ -482,24 +546,6 @@ def delete_engine_download():
(f"Error deleting {json_data.get('engine')} {json_data.get('version')}", 500)
@server.get('/api/engine/<engine_name>/args')
def get_engine_args(engine_name):
try:
engine_class = EngineManager.engine_with_name(engine_name)
return engine_class().get_arguments()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
@server.get('/api/engine/<engine_name>/help')
def get_engine_help(engine_name):
try:
engine_class = EngineManager.engine_with_name(engine_name)
return engine_class().get_help()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
# --------------------------------------------
# Miscellaneous:
# --------------------------------------------

View File

@@ -247,16 +247,15 @@ class RenderServerProxy:
# Engines:
# --------------------------------------------
def is_engine_available(self, engine_name):
return self.request_data(f'{engine_name}/is_available')
def get_installed_engines(self, timeout=5):
return self.request_data(f'installed_engines', timeout)
def get_all_engines(self):
# todo: this doesnt work
return self.request_data('all_engines')
def is_engine_available(self, engine_name:str, timeout=5):
return self.request_data(f'{engine_name}/is_available', timeout)
def get_engine_info(self, response_type='standard', timeout=5):
def get_all_engine_info(self, response_type='standard', timeout=5):
"""
Fetches engine information from the server.
Fetches all engine information from the server.
Args:
response_type (str, optional): Returns standard or full version of engine info
@@ -268,19 +267,33 @@ class RenderServerProxy:
all_data = self.request_data(f"engine_info?response_type={response_type}", timeout=timeout)
return all_data
def delete_engine(self, engine, version, system_cpu=None):
def get_engine_info(self, engine_name:str, response_type='standard', timeout=5):
"""
Fetches specific engine information from the server.
Args:
engine_name (str): Name of the engine
response_type (str, optional): Returns standard or full version of engine info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the engine information.
"""
return self.request_data(f'{engine_name}/info?response_type={response_type}', timeout)
def delete_engine(self, engine_name:str, version:str, system_cpu=None):
"""
Sends a request to the server to delete a specific engine.
Args:
engine (str): The name of the engine to delete.
engine_name (str): The name of the engine to delete.
version (str): The version of the engine to delete.
system_cpu (str, optional): The system CPU type. Defaults to None.
Returns:
Response: The response from the server.
"""
form_data = {'engine': engine, 'version': version, 'system_cpu': system_cpu}
form_data = {'engine': engine_name, 'version': version, 'system_cpu': system_cpu}
return requests.post(f'http://{self.hostname}:{self.port}/api/delete_engine', json=form_data)
# --------------------------------------------

View File

@@ -1,6 +1,10 @@
import bpy
import json
# Force CPU rendering
bpy.context.preferences.addons["cycles"].preferences.compute_device_type = "NONE"
bpy.context.scene.cycles.device = "CPU"
# Ensure Cycles is available
bpy.context.preferences.addons['cycles'].preferences.get_devices()

View File

@@ -4,7 +4,7 @@ import platform
import subprocess
logger = logging.getLogger()
SUBPROCESS_TIMEOUT = 5
SUBPROCESS_TIMEOUT = 10
class BaseRenderEngine(object):

View File

@@ -3,17 +3,17 @@ import os
import shutil
import threading
import concurrent.futures
from typing import Type
from engines.core.base_engine import BaseRenderEngine
from src.engines.blender.blender_engine import Blender
from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu
logger = logging.getLogger()
ENGINE_CLASSES = [Blender, FFMPEG]
class EngineManager:
"""Class that manages different versions of installed render engines and handles fetching and downloading new versions,
if possible.
@@ -23,37 +23,37 @@ class EngineManager:
download_tasks = []
@staticmethod
def supported_engines():
def supported_engines() -> list[type[BaseRenderEngine]]:
return ENGINE_CLASSES
# --- Installed Engines ---
@classmethod
def engine_for_project_path(cls, path):
def engine_class_for_project_path(cls, path) -> type[BaseRenderEngine]:
_, extension = os.path.splitext(path)
extension = extension.lower().strip('.')
for engine_class in cls.supported_engines():
engine = cls.get_latest_engine_instance(engine_class)
if extension in engine.supported_extensions():
return engine
return engine_class
undefined_renderer_support = [x for x in cls.supported_engines() if not cls.get_latest_engine_instance(x).supported_extensions()]
return undefined_renderer_support[0]
@classmethod
def engine_with_name(cls, engine_name):
def engine_class_with_name(cls, engine_name: str) -> type[BaseRenderEngine] | None:
for obj in cls.supported_engines():
if obj.name().lower() == engine_name.lower():
return obj
return None
@classmethod
def get_latest_engine_instance(cls, engine_class):
def get_latest_engine_instance(cls, engine_class: Type[BaseRenderEngine]) -> BaseRenderEngine:
newest = cls.newest_installed_engine_data(engine_class.name())
engine = engine_class(newest["path"])
return engine
@classmethod
def get_installed_engine_data(cls, filter_name=None, include_corrupt=False, ignore_system=False):
def get_installed_engine_data(cls, filter_name=None, include_corrupt=False, ignore_system=False) -> list:
if not cls.engines_path:
raise FileNotFoundError("Engine path is not set")
@@ -75,7 +75,7 @@ class EngineManager:
# Initialize binary_name with engine name
binary_name = result_dict['engine'].lower()
# Determine the correct binary name based on the engine and system_os
eng = cls.engine_with_name(result_dict['engine'])
eng = cls.engine_class_with_name(result_dict['engine'])
binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
# Find the path to the binary file
@@ -141,13 +141,13 @@ class EngineManager:
cls.download_engine(engine.name(), update_available['version'], background=True)
@classmethod
def all_version_data_for_engine(cls, engine_name, include_corrupt=False, ignore_system=False):
def all_version_data_for_engine(cls, engine_name:str, include_corrupt=False, ignore_system=False) -> list:
versions = cls.get_installed_engine_data(filter_name=engine_name, include_corrupt=include_corrupt, ignore_system=ignore_system)
sorted_versions = sorted(versions, key=lambda x: x['version'], reverse=True)
return sorted_versions
@classmethod
def newest_installed_engine_data(cls, engine_name, system_os=None, cpu=None, ignore_system=None):
def newest_installed_engine_data(cls, engine_name:str, system_os=None, cpu=None, ignore_system=None) -> list:
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
@@ -157,37 +157,37 @@ class EngineManager:
return filtered[0]
except IndexError:
logger.error(f"Cannot find newest engine version for {engine_name}-{system_os}-{cpu}")
return None
return []
@classmethod
def is_version_installed(cls, engine, version, system_os=None, cpu=None, ignore_system=False):
def is_version_installed(cls, engine_name:str, version, system_os=None, cpu=None, ignore_system=False):
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
filtered = [x for x in cls.get_installed_engine_data(filter_name=engine, ignore_system=ignore_system) if
filtered = [x for x in cls.get_installed_engine_data(filter_name=engine_name, ignore_system=ignore_system) if
x['system_os'] == system_os and x['cpu'] == cpu and x['version'] == version]
return filtered[0] if filtered else False
@classmethod
def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None):
def version_is_available_to_download(cls, engine_name:str, version, system_os=None, cpu=None):
try:
downloader = cls.engine_with_name(engine).downloader()
downloader = cls.engine_class_with_name(engine_name).downloader()
return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in version_is_available_to_download: {e}")
return None
@classmethod
def find_most_recent_version(cls, engine=None, system_os=None, cpu=None, lts_only=False):
def find_most_recent_version(cls, engine_name:str, system_os=None, cpu=None, lts_only=False) -> dict:
try:
downloader = cls.engine_with_name(engine).downloader()
downloader = cls.engine_class_with_name(engine_name).downloader()
return downloader.find_most_recent_version(system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in find_most_recent_version: {e}")
return None
return {}
@classmethod
def is_engine_update_available(cls, engine_class, ignore_system_installs=False):
def is_engine_update_available(cls, engine_class: Type[BaseRenderEngine], ignore_system_installs=False):
logger.debug(f"Checking for updates to {engine_class.name()}")
latest_version = engine_class.downloader().find_most_recent_version()
@@ -209,23 +209,23 @@ class EngineManager:
return [engine for engine in cls.supported_engines() if hasattr(engine, "downloader") and engine.downloader()]
@classmethod
def get_existing_download_task(cls, engine, version, system_os=None, cpu=None):
def get_existing_download_task(cls, engine_name, version, system_os=None, cpu=None):
for task in cls.download_tasks:
task_parts = task.name.split('-')
task_engine, task_version, task_system_os, task_cpu = task_parts[:4]
if engine == task_engine and version == task_version:
if engine_name == task_engine and version == task_version:
if system_os in (task_system_os, None) and cpu in (task_cpu, None):
return task
return None
@classmethod
def download_engine(cls, engine, version, system_os=None, cpu=None, background=False, ignore_system=False):
def download_engine(cls, engine_name, version, system_os=None, cpu=None, background=False, ignore_system=False):
engine_to_download = cls.engine_with_name(engine)
existing_task = cls.get_existing_download_task(engine, version, system_os, cpu)
engine_to_download = cls.engine_class_with_name(engine_name)
existing_task = cls.get_existing_download_task(engine_name, version, system_os, cpu)
if existing_task:
logger.debug(f"Already downloading {engine} {version}")
logger.debug(f"Already downloading {engine_name} {version}")
if not background:
existing_task.join() # If download task exists, wait until it's done downloading
return None
@@ -235,7 +235,7 @@ class EngineManager:
elif not cls.engines_path:
raise FileNotFoundError("Engines path must be set before requesting downloads")
thread = EngineDownloadWorker(engine, version, system_os, cpu)
thread = EngineDownloadWorker(engine_name, version, system_os, cpu)
cls.download_tasks.append(thread)
thread.start()
@@ -243,29 +243,29 @@ class EngineManager:
return thread
thread.join()
found_engine = cls.is_version_installed(engine, version, system_os, cpu, ignore_system) # Check that engine downloaded
found_engine = cls.is_version_installed(engine_name, version, system_os, cpu, ignore_system) # Check that engine downloaded
if not found_engine:
logger.error(f"Error downloading {engine}")
logger.error(f"Error downloading {engine_name}")
return found_engine
@classmethod
def delete_engine_download(cls, engine, version, system_os=None, cpu=None):
logger.info(f"Requested deletion of engine: {engine}-{version}")
def delete_engine_download(cls, engine_name, version, system_os=None, cpu=None):
logger.info(f"Requested deletion of engine: {engine_name}-{version}")
found = cls.is_version_installed(engine, version, system_os, cpu)
found = cls.is_version_installed(engine_name, version, system_os, cpu)
if found and found['type'] == 'managed': # don't delete system installs
# find the root directory of the engine executable
root_dir_name = '-'.join([engine, version, found['system_os'], found['cpu']])
root_dir_name = '-'.join([engine_name, version, found['system_os'], found['cpu']])
remove_path = os.path.join(found['path'].split(root_dir_name)[0], root_dir_name)
# delete the file path
logger.info(f"Deleting engine at path: {remove_path}")
shutil.rmtree(remove_path, ignore_errors=False)
logger.info(f"Engine {engine}-{version}-{found['system_os']}-{found['cpu']} successfully deleted")
logger.info(f"Engine {engine_name}-{version}-{found['system_os']}-{found['cpu']} successfully deleted")
return True
elif found: # these are managed by the system / user. Don't delete these.
logger.error(f'Cannot delete requested {engine} {version}. Managed externally.')
logger.error(f'Cannot delete requested {engine_name} {version}. Managed externally.')
else:
logger.error(f"Cannot find engine: {engine}-{version}")
logger.error(f"Cannot find engine: {engine_name}-{version}")
return False
# --- Background Tasks ---
@@ -277,7 +277,7 @@ class EngineManager:
@classmethod
def create_worker(cls, engine_name, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
worker_class = cls.engine_with_name(engine_name).worker_class()
worker_class = cls.engine_class_with_name(engine_name).worker_class()
# check to make sure we have versions installed
all_versions = cls.all_version_data_for_engine(engine_name)
@@ -342,7 +342,7 @@ class EngineDownloadWorker(threading.Thread):
return existing_download
# Get the appropriate downloader class based on the engine type
downloader = EngineManager.engine_with_name(self.engine).downloader()
downloader = EngineManager.engine_class_with_name(self.engine).downloader()
downloader.download_engine( self.version, download_location=EngineManager.engines_path,
system_os=self.system_os, cpu=self.cpu, timeout=300, progress_callback=self._update_progress)
except Exception as e:

View File

@@ -63,7 +63,6 @@ class NewRenderJobForm(QWidget):
# Job / Server Data
self.server_proxy = RenderServerProxy(socket.gethostname())
self.engine_info = None
self.project_info = None
# Setup
@@ -107,6 +106,8 @@ class NewRenderJobForm(QWidget):
job_name_layout.addWidget(QLabel("Job name:"))
self.job_name_input = QLineEdit()
job_name_layout.addWidget(self.job_name_input)
self.engine_type = QComboBox()
job_name_layout.addWidget(self.engine_type)
file_group_layout.addLayout(job_name_layout)
# Job File
@@ -242,12 +243,7 @@ class NewRenderJobForm(QWidget):
engine_group_layout = QVBoxLayout(self.engine_group)
engine_layout = QHBoxLayout()
engine_layout.addWidget(QLabel("Engine:"))
self.engine_type = QComboBox()
self.engine_type.currentIndexChanged.connect(self.engine_changed)
engine_layout.addWidget(self.engine_type)
engine_layout.addWidget(QLabel("Version:"))
engine_layout.addWidget(QLabel("Engine Version:"))
self.engine_version_combo = QComboBox()
self.engine_version_combo.addItem('latest')
engine_layout.addWidget(self.engine_version_combo)
@@ -329,10 +325,10 @@ class NewRenderJobForm(QWidget):
def update_engine_info(self):
# get the engine info and add them all to the ui
self.engine_info = self.server_proxy.get_engine_info(response_type='full')
self.engine_type.addItems(self.engine_info.keys())
engine = EngineManager.engine_class_for_project_path(self.project_path)
installed_engines = self.server_proxy.get_installed_engines()
self.engine_type.addItems(installed_engines.keys())
# select the best engine for the file type
engine = EngineManager.engine_for_project_path(self.project_path)
self.engine_type.setCurrentText(engine.name().lower())
# refresh ui
self.engine_changed()
@@ -344,9 +340,12 @@ class NewRenderJobForm(QWidget):
self.engine_version_combo.addItem('latest')
self.file_format_combo.clear()
if current_engine:
engine_vers = [version_info['version'] for version_info in self.engine_info[current_engine]['versions']]
engine_info = self.server_proxy.get_engine_info(current_engine, 'full', timeout=10)
if not engine_info:
raise FileNotFoundError(f"Cannot get information about engine '{current_engine}'")
engine_vers = [v['version'] for v in engine_info['versions']]
self.engine_version_combo.addItems(engine_vers)
self.file_format_combo.addItems(self.engine_info[current_engine]['supported_export_formats'])
self.file_format_combo.addItems(engine_info.get('supported_export_formats'))
def update_server_list(self):
clients = ZeroconfServer.found_hostnames()
@@ -373,6 +372,7 @@ class NewRenderJobForm(QWidget):
# setup bg worker
self.worker_thread = GetProjectInfoWorker(window=self, project_path=file_name)
self.worker_thread.message_signal.connect(self.post_get_project_info_update)
self.worker_thread.error_signal.connect(self.show_error_message)
self.worker_thread.start()
def browse_output_path(self):
@@ -386,6 +386,13 @@ class NewRenderJobForm(QWidget):
self.engine_help_viewer = EngineHelpViewer(url)
self.engine_help_viewer.show()
def show_error_message(self, message):
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Icon.Critical)
msg.setWindowTitle("Error")
msg.setText(message)
msg.exec()
# -------- Update --------
def post_get_project_info_update(self):
@@ -393,7 +400,7 @@ class NewRenderJobForm(QWidget):
try:
# Set the best engine we can find
input_path = self.scene_file_input.text()
engine = EngineManager.engine_for_project_path(input_path)
engine = EngineManager.engine_class_for_project_path(input_path)
engine_index = self.engine_type.findText(engine.name().lower())
if engine_index >= 0:
@@ -435,7 +442,7 @@ class NewRenderJobForm(QWidget):
# Dynamic Engine Options
clear_layout(self.engine_options_layout) # clear old options
# dynamically populate option list
self.current_engine_options = engine().ui_options()
self.current_engine_options = {} #todo: fix this
for option in self.current_engine_options:
h_layout = QHBoxLayout()
label = QLabel(option['name'].replace('_', ' ').capitalize() + ':')
@@ -586,8 +593,9 @@ class SubmitWorker(QThread):
job_json['child_jobs'] = children_jobs
# presubmission tasks
engine = EngineManager.engine_with_name(self.window.engine_type.currentText().lower())
input_path = engine().perform_presubmission_tasks(input_path)
engine_class = EngineManager.engine_class_with_name(self.window.engine_type.currentText().lower())
latest_engine = EngineManager.get_latest_engine_instance(engine_class)
input_path = latest_engine.perform_presubmission_tasks(input_path)
# submit
err_msg = ""
result = self.window.server_proxy.post_job_to_server(file_path=input_path, job_data=job_json,
@@ -605,6 +613,7 @@ class GetProjectInfoWorker(QThread):
"""Worker class called to retrieve information about a project file on a background thread and update the UI"""
message_signal = pyqtSignal()
error_signal = pyqtSignal(str)
def __init__(self, window, project_path):
super().__init__()
@@ -612,9 +621,14 @@ class GetProjectInfoWorker(QThread):
self.project_path = project_path
def run(self):
engine = EngineManager.engine_for_project_path(self.project_path)
self.window.project_info = engine().get_project_info(self.project_path)
self.message_signal.emit()
try:
self.window.update_engine_info()
engine_class = EngineManager.engine_class_for_project_path(self.project_path)
engine = EngineManager.get_latest_engine_instance(engine_class)
self.window.project_info = engine.get_project_info(self.project_path)
self.message_signal.emit()
except Exception as e:
self.error_signal.emit(str(e))
def clear_layout(layout):

View File

@@ -93,7 +93,7 @@ class EngineBrowserWindow(QMainWindow):
def update_table(self):
def update_table_worker():
raw_server_data = RenderServerProxy(self.hostname).get_engine_info()
raw_server_data = RenderServerProxy(self.hostname).get_all_engine_info()
if not raw_server_data:
return

View File

@@ -37,7 +37,7 @@ class GetEngineInfoWorker(QThread):
self.parent = parent
def run(self):
data = RenderServerProxy(socket.gethostname()).get_engine_info()
data = RenderServerProxy(socket.gethostname()).get_all_engine_info()
self.done.emit(data)
class SettingsWindow(QMainWindow):
@@ -413,7 +413,7 @@ class SettingsWindow(QMainWindow):
msg_result = msg_box.exec()
messagebox_shown = True
if msg_result == QMessageBox.StandardButton.Yes:
EngineManager.download_engine(engine=engine.name(), version=result['version'], background=True,
EngineManager.download_engine(engine_name=engine.name(), version=result['version'], background=True,
ignore_system=ignore_system)
self.engine_download_progress_bar.setHidden(False)
self.engine_download_progress_bar.setValue(0)