Report installed renderers in status

requirements.txt

@@ -1,4 +1,5 @@
 requests==2.31.0
+requests_toolbelt==1.0.0
 psutil==5.9.6
 PyYAML==6.0.1
 Flask==3.0.0

src/api_server.py

@@ -7,6 +7,7 @@ import platform
 import shutil
 import socket
 import ssl
+import tempfile
 import threading
 import time
 import zipfile
@@ -26,6 +27,7 @@ from src.render_queue import RenderQueue, JobNotFoundError
 from src.server_proxy import RenderServerProxy
 from src.utilities.server_helper import generate_thumbnail_for_job
 from src.utilities.zeroconf_server import ZeroconfServer
+from src.utilities.misc_helper import system_safe_path
 from src.workers.worker_factory import RenderWorkerFactory
 from src.workers.base_worker import string_to_status, RenderStatus
 
@@ -54,7 +56,7 @@ def sorted_jobs(all_jobs, sort_by_date=True):
 @server.route('/')
 @server.route('/index')
 def index():
-    with open('config/presets.yaml') as f:
+    with open(system_safe_path('config/presets.yaml')) as f:
         render_presets = yaml.load(f, Loader=yaml.FullLoader)
 
     return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()),
@@ -188,7 +190,7 @@ def get_job_status(job_id):
 @server.get('/api/job/<job_id>/logs')
 def get_job_logs(job_id):
     found_job = RenderQueue.job_with_id(job_id)
-    log_path = found_job.log_path()
+    log_path = system_safe_path(found_job.log_path()),
     log_data = None
     if log_path and os.path.exists(log_path):
         with open(log_path) as file:
@@ -232,10 +234,11 @@ def download_all(job_id):
     found_job = RenderQueue.job_with_id(job_id)
     output_dir = os.path.dirname(found_job.output_path)
     if os.path.exists(output_dir):
-        zip_filename = os.path.join('/tmp', pathlib.Path(found_job.input_path).stem + '.zip')
+        zip_filename = system_safe_path(os.path.join(tempfile.gettempdir(),
+                                                     pathlib.Path(found_job.input_path).stem + '.zip'))
         with ZipFile(zip_filename, 'w') as zipObj:
             for f in os.listdir(output_dir):
-                zipObj.write(filename=os.path.join(output_dir, f),
+                zipObj.write(filename=system_safe_path(os.path.join(output_dir, f)),
                              arcname=os.path.basename(f))
         return send_file(zip_filename, mimetype="zip", as_attachment=True, )
     else:
@@ -244,7 +247,8 @@ def download_all(job_id):
 
 @server.get('/api/presets')
 def presets():
-    with open('config/presets.yaml') as f:
+    presets_path = system_safe_path('config/presets.yaml')
+    with open(presets_path) as f:
         presets = yaml.load(f, Loader=yaml.FullLoader)
     return presets
 
@@ -387,7 +391,7 @@ def add_job_handler():
             try:
                 # prepare output paths
                 output_dir = os.path.join(job_dir, job_data.get('name') if len(jobs_list) > 1 else 'output')
-                os.makedirs(output_dir, exist_ok=True)
+                os.makedirs(system_safe_path(output_dir), exist_ok=True)
 
                 # get new output path in output_dir
                 job_data['output_path'] = os.path.join(output_dir, os.path.basename(
@@ -398,6 +402,7 @@ def add_job_handler():
                 worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'],
                                                            input_path=loaded_project_local_path,
                                                            output_path=job_data["output_path"],
+                                                           engine_version=job_data.get('engine_version'),
                                                            args=job_data.get('args', {}))
                 worker.status = job_data.get("initial_status", worker.status)
                 worker.parent = job_data.get("parent", worker.parent)
@@ -500,7 +505,7 @@ def clear_history():
 def status():
     renderer_data = {}
     for render_class in RenderWorkerFactory.supported_classes():
-        if render_class.engine.default_renderer_path():  # only return renderers installed on host
+        if EngineManager.all_versions_for_engine(render_class.name):  # only return renderers installed on host
             renderer_data[render_class.engine.name()] = \
                 {'versions': EngineManager.all_versions_for_engine(render_class.engine.name()),
                  'is_available': RenderQueue.is_available_for_job(render_class.engine.name())
@@ -533,13 +538,15 @@ def renderer_info():
     renderer_data = {}
     for engine_name in RenderWorkerFactory.supported_renderers():
         engine = RenderWorkerFactory.class_for_name(engine_name).engine
-        if engine.default_renderer_path():
 
-            # Get all installed versions of engine
+        # Get all installed versions of engine
+        installed_versions = EngineManager.all_versions_for_engine(engine_name)
+        if installed_versions:
+            install_path = installed_versions[0]['path']
             renderer_data[engine_name] = {'is_available': RenderQueue.is_available_for_job(engine.name()),
-                                'versions': EngineManager.all_versions_for_engine(engine_name),
+                                'versions': installed_versions,
                                 'supported_extensions': engine.supported_extensions,
-                                'supported_export_formats': engine().get_output_formats()}
+                                'supported_export_formats': engine(install_path).get_output_formats()}
     return renderer_data
 
 
@@ -581,7 +588,7 @@ def start_server(background_thread=False):
             RenderQueue.evaluate_queue()
             time.sleep(delay_sec)
 
-    with open('config/config.yaml') as f:
+    with open(system_safe_path('config/config.yaml')) as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
 
     logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
@@ -594,15 +601,20 @@ def start_server(background_thread=False):
     # load flask settings
     server.config['HOSTNAME'] = local_hostname
     server.config['PORT'] = int(config.get('port_number', 8080))
-    server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder'])
-    server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs')
+    server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(config['upload_folder']))
+    server.config['THUMBS_FOLDER'] = system_safe_path(os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs'))
     server.config['MAX_CONTENT_PATH'] = config['max_content_path']
     server.config['enable_split_jobs'] = config.get('enable_split_jobs', False)
 
     # Setup directory for saving engines to
-    EngineManager.engines_path = os.path.join(os.path.join(os.path.expanduser(config['upload_folder']), 'engines'))
+    EngineManager.engines_path = system_safe_path(os.path.join(os.path.join(os.path.expanduser(config['upload_folder']), 'engines')))
     os.makedirs(EngineManager.engines_path, exist_ok=True)
 
+    # Debug info
+    logger.debug(f"Upload directory: {server.config['UPLOAD_FOLDER']}")
+    logger.debug(f"Thumbs directory: {server.config['THUMBS_FOLDER']}")
+    logger.debug(f"Engines directory: {EngineManager.engines_path}")
+
     # disable most Flask logging
     flask_log = logging.getLogger('werkzeug')
     flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper())

src/engines/base_engine.py

@@ -13,6 +13,8 @@ class BaseRenderEngine(object):
 
     def __init__(self, custom_path=None):
         self.custom_renderer_path = custom_path
+        if not self.renderer_path():
+            raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance")
 
     def renderer_path(self):
         return self.custom_renderer_path or self.default_renderer_path()
@@ -24,9 +26,9 @@ class BaseRenderEngine(object):
     @classmethod
     def default_renderer_path(cls):
         path = None
-        try:
+        try:  # Linux and macOS
             path = subprocess.check_output(['which', cls.name()], timeout=SUBPROCESS_TIMEOUT).decode('utf-8').strip()
-        except subprocess.CalledProcessError:
+        except (subprocess.CalledProcessError, FileNotFoundError):
             for p in cls.install_paths:
                 if os.path.exists(p):
                     path = p

src/engines/blender_engine.py

@@ -13,6 +13,7 @@ class Blender(BaseRenderEngine):
 
     install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender']
     supported_extensions = ['.blend']
+    binary_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender'}
 
     def version(self):
         version = None
@@ -36,18 +37,17 @@ class Blender(BaseRenderEngine):
                 return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression],
                                       capture_output=True, timeout=timeout)
             except Exception as e:
-                logger.warning(f"Error running python expression in blender: {e}")
-                pass
+                logger.error(f"Error running python expression in blender: {e}")
         else:
             raise FileNotFoundError(f'Project file not found: {project_path}')
 
     def run_python_script(self, project_path, script_path, timeout=None):
         if os.path.exists(project_path) and os.path.exists(script_path):
             try:
-                return subprocess.run([self.default_renderer_path(), '-b', project_path, '--python', script_path],
+                return subprocess.run([self.renderer_path(), '-b', project_path, '--python', script_path],
                                       capture_output=True, timeout=timeout)
             except Exception as e:
-                logger.warning(f"Error running python expression in blender: {e}")
+                logger.warning(f"Error running python script in blender: {e}")
                 pass
         elif not os.path.exists(project_path):
             raise FileNotFoundError(f'Project file not found: {project_path}')
@@ -58,8 +58,9 @@ class Blender(BaseRenderEngine):
     def get_scene_info(self, project_path, timeout=10):
         scene_info = {}
         try:
-            results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                                                        'scripts', 'blender', 'get_file_info.py'), timeout=timeout)
+            script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                       'scripts', 'blender', 'get_file_info.py')
+            results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout)
             result_text = results.stdout.decode()
             for line in result_text.splitlines():
                 if line.startswith('SCENE_DATA:'):
@@ -76,8 +77,9 @@ class Blender(BaseRenderEngine):
         # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935
         try:
             logger.info(f"Starting to pack Blender file: {project_path}")
-            results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                                                       'scripts', 'blender', 'pack_project.py'), timeout=timeout)
+            script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                       'scripts', 'blender', 'pack_project.py')
+            results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout)
 
             result_text = results.stdout.decode()
             dir_name = os.path.dirname(project_path)

src/engines/downloaders/blender_downloader.py

@@ -53,7 +53,7 @@ class BlenderDownloader:
             versions_data = [x for x in versions_data if x['cpu'] == cpu]
 
         for v in versions_data:
-            v['url'] = os.path.join(base_url, v['file'])
+            v['url'] = base_url + '/' + v['file']
 
         versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
         return versions_data

src/engines/downloaders/ffmpeg_downloader.py

@@ -87,7 +87,7 @@ class FFMPEGDownloader:
             release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases'
             remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz')
         elif system_os == 'windows':
-            remote_url = os.path.join(cls.windows_download_url, version, f'ffmpeg-{version}-full_build.zip')
+            remote_url = f"{cls.windows_download_url.strip('/')}/{version}/ffmpeg-{version}-full_build.zip"
 
         # Download and extract
         try:

src/engines/engine_manager.py

@@ -4,6 +4,7 @@ import platform
 import shutil
 from .downloaders.blender_downloader import BlenderDownloader
 from .downloaders.ffmpeg_downloader import FFMPEGDownloader
+from ..utilities.misc_helper import system_safe_path
 
 try:
     from .blender_engine import Blender
@@ -37,14 +38,25 @@ class EngineManager:
                 # Split the input string by dashes to get segments
                 segments = directory.split('-')
 
-                # Define the keys for each word
-                keys = ["engine", "version", "system_os", "cpu"]
-
                 # Create a dictionary with named keys
-                executable_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender.app'}
+                keys = ["engine", "version", "system_os", "cpu"]
                 result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))}
-                result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names.get(result_dict['system_os'], 'unknown'))
                 result_dict['type'] = 'managed'
+
+                # Figure out the binary name for the path
+                binary_name = result_dict['engine'].lower()
+                for eng in cls.supported_engines():
+                    if eng.name().lower() == result_dict['engine']:
+                        binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
+
+                # Find path to binary
+                path = None
+                for root, _, files in os.walk(system_safe_path(os.path.join(cls.engines_path, directory))):
+                    if binary_name in files:
+                        path = os.path.join(root, binary_name)
+                        break
+
+                result_dict['path'] = path
                 results.append(result_dict)
         except FileNotFoundError:
             logger.warning("Cannot find local engines download directory")
@@ -113,11 +125,15 @@ class EngineManager:
             return
 
         # Get the appropriate downloader class based on the engine type
-        downloader = downloader_classes[engine]
-        if downloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu):
-            return cls.has_engine_version(engine, version, system_os, cpu)
-        else:
+        downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
+                                                   system_os=system_os, cpu=cpu)
+
+        # Check that engine was properly downloaded
+        found_engine = cls.has_engine_version(engine, version, system_os, cpu)
+        if not found_engine:
             logger.error(f"Error downloading {engine}")
+        return found_engine
+
 
     @classmethod
     def delete_engine_download(cls, engine, version, system_os=None, cpu=None):

src/engines/ffmpeg_engine.py

@@ -7,6 +7,8 @@ import re
 
 class FFMPEG(BaseRenderEngine):
 
+    binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
+
     def version(self):
         version = None
         try:
@@ -31,11 +33,15 @@ class FFMPEG(BaseRenderEngine):
         return encoders
 
     def get_all_formats(self):
-        formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
-                                             timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
-        pattern = '(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)'
-        all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
-        return all_formats
+        try:
+            formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
+                                                 timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
+            pattern = '(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)\r'
+            all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
+            return all_formats
+        except Exception as e:
+            logger.error(f"Error getting all formats: {e}")
+        return []
 
     def extension_for_format(self, ffmpeg_format):
         # Extract the common extension using regex
@@ -53,7 +59,7 @@ class FFMPEG(BaseRenderEngine):
         return [x for x in self.get_all_formats() if 'E' in x['type'].upper()]
 
     def get_frame_count(self, path_to_file):
-        raw_stdout = subprocess.check_output([self.default_renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
+        raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
                                                '-f', 'null', '-'], stderr=subprocess.STDOUT,
                                               timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
         match = re.findall(r'frame=\s*(\d+)', raw_stdout)

src/render_queue.py

@@ -5,7 +5,7 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from src.utilities.status_utils import RenderStatus
-from src.workers.worker_factory import RenderWorkerFactory
+from src.engines.engine_manager import EngineManager
 from src.workers.base_worker import Base
 
 logger = logging.getLogger()
@@ -100,7 +100,8 @@ class RenderQueue:
 
     @classmethod
     def is_available_for_job(cls, renderer, priority=2):
-        if not RenderWorkerFactory.class_for_name(renderer).engine.default_renderer_path():
+
+        if not EngineManager.all_versions_for_engine(renderer):
             return False
 
         instances = cls.renderer_instances()

src/utilities/misc_helper.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import platform
 import subprocess
 from datetime import datetime
 
@@ -103,3 +104,9 @@ def get_file_size_human(file_path):
     else:
         return f"{size_in_bytes / 1024 ** 4:.2f} TB"
 
+
+# Convert path to the appropriate format for the current platform
+def system_safe_path(path):
+    if platform.system().lower() == "windows":
+        return os.path.normpath(path)
+    return path.replace("\\", "/")

src/workers/base_worker.py

@@ -30,6 +30,7 @@ class BaseRenderWorker(Base):
     end_time = Column(DateTime, nullable=True)
     renderer = Column(String)
     renderer_version = Column(String)
+    renderer_path = Column(String)
     priority = Column(Integer)
     project_length = Column(Integer)
     start_frame = Column(Integer)
@@ -42,7 +43,7 @@ class BaseRenderWorker(Base):
 
     engine = None
 
-    def __init__(self, input_path, output_path, priority=2, args=None, ignore_extensions=True, parent=None,
+    def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None,
                  name=None):
 
         if not ignore_extensions:
@@ -64,7 +65,8 @@ class BaseRenderWorker(Base):
         self.args = args or {}
         self.date_created = datetime.now()
         self.renderer = self.engine.name()
-        self.renderer_version = self.engine().version()
+        self.renderer_path = engine_path
+        self.renderer_version = self.engine(engine_path).version()
         self.custom_renderer_path = None
         self.priority = priority
         self.parent = parent
@@ -159,7 +161,7 @@ class BaseRenderWorker(Base):
             self.errors.append(msg)
             return
 
-        if not self.engine.default_renderer_path() and not self.custom_renderer_path:
+        if not os.path.exists(self.renderer_path):
             self.status = RenderStatus.ERROR
             msg = 'Cannot find render engine path for {}'.format(self.engine.name())
             logger.error(msg)
@@ -168,7 +170,7 @@ class BaseRenderWorker(Base):
 
         self.status = RenderStatus.RUNNING
         self.start_time = datetime.now()
-        logger.info(f'Starting {self.engine.name()} {self.engine().version()} Render for {self.input_path} | '
+        logger.info(f'Starting {self.engine.name()} {self.renderer_version} Render for {self.input_path} | '
                     f'Frame Count: {self.total_frames}')
         self.__thread.start()
 
@@ -183,7 +185,7 @@ class BaseRenderWorker(Base):
 
         with open(self.log_path(), "a") as f:
 
-            f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine().version()} "
+            f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.renderer_version} "
                     f"render for {self.input_path}\n\n")
             f.write(f"Running command: {subprocess_cmds}\n")
             f.write('=' * 80 + '\n\n')

src/workers/blender_worker.py

@@ -11,9 +11,9 @@ class BlenderRenderWorker(BaseRenderWorker):
 
     engine = Blender
 
-    def __init__(self, input_path, output_path, args=None, parent=None, name=None):
-        super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args,
-                                                  parent=parent, name=name)
+    def __init__(self, input_path, output_path, engine_path, args=None, parent=None, name=None):
+        super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path,
+                                                  engine_path=engine_path, args=args, parent=parent, name=name)
 
         # Args
         self.blender_engine = self.args.get('engine', 'BLENDER_EEVEE').upper()
@@ -24,7 +24,7 @@ class BlenderRenderWorker(BaseRenderWorker):
         self.__frame_percent_complete = 0.0
 
         # Scene Info
-        self.scene_info = Blender().get_scene_info(input_path)
+        self.scene_info = Blender(engine_path).get_scene_info(input_path)
         self.start_frame = int(self.scene_info.get('start_frame', 1))
         self.end_frame = int(self.scene_info.get('end_frame', self.start_frame))
         self.project_length = (self.end_frame - self.start_frame) + 1
@@ -32,7 +32,7 @@ class BlenderRenderWorker(BaseRenderWorker):
 
     def generate_worker_subprocess(self):
 
-        cmd = [self.engine.default_renderer_path()]
+        cmd = [self.renderer_path]
         if self.args.get('background', True):  # optionally run render not in background
             cmd.append('-b')
         cmd.append(self.input_path)

src/workers/ffmpeg_worker.py

@@ -14,7 +14,7 @@ class FFMPEGRenderWorker(BaseRenderWorker):
         super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args,
                                                  parent=parent, name=name)
 
-        stream_info = subprocess.check_output([self.engine.default_renderer_path(), "-i",  # https://stackoverflow.com/a/61604105
+        stream_info = subprocess.check_output([self.renderer_path, "-i",  # https://stackoverflow.com/a/61604105
                                                input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y",
                                                      "/dev/null"], stderr=subprocess.STDOUT).decode('utf-8')
         found_frames = re.findall('frame=\s*(\d+)', stream_info)

src/workers/worker_factory.py

@@ -1,3 +1,9 @@
+import logging
+from ..engines.engine_manager import EngineManager
+
+logger = logging.getLogger()
+
+
 class RenderWorkerFactory:
 
     @staticmethod
@@ -10,9 +16,26 @@ class RenderWorkerFactory:
         return classes
 
     @staticmethod
-    def create_worker(renderer, input_path, output_path, args=None, parent=None, name=None):
+    def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
+
         worker_class = RenderWorkerFactory.class_for_name(renderer)
-        return worker_class(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name)
+
+        # find correct engine version
+        all_versions = EngineManager.all_versions_for_engine(renderer)
+        if not all_versions:
+            raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
+
+        engine_path = all_versions[0]['path']
+        if engine_version:
+            for ver in all_versions:
+                if ver['version'] == engine_version:
+                    engine_path = ver['path']
+                    break
+            if not engine_path:
+                logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}")
+
+        return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
+                            parent=parent, name=name)
 
     @staticmethod
     def supported_renderers():