Mac try again

Add lots of docstrings everywhere

.github/workflows/create-executables.yml

@@ -0,0 +1,67 @@
+name: Create Executables
+            
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - created
+  push:
+    branches:
+       - github-actions
+
+jobs:
+  # pyinstaller-build-windows:
+  #   runs-on: windows-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
+  #     - name: Extract Version
+  #       id: get_version
+  #       run: |
+  #         $version = python -c "from version import APP_VERSION; print(APP_VERSION)"
+  #         echo "VERSION=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8
+  #     - name: Create Executable (Windows-x64)
+  #       uses: sayyid5416/pyinstaller@v1
+  #       with:
+  #         python_ver: '3.11'
+  #         python_arch: 'x64'
+  #         spec: 'main.spec'
+  #         requirements: 'requirements.txt'
+  #         upload_exe_with_name: 'Zordon-${{env.VERSION}}-Windows-x64'
+  # pyinstaller-build-linux:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
+  #     - name: Extract Version
+  #       id: get_version
+  #       run: |
+  #         version=$(python -c 'from version import APP_VERSION; print(APP_VERSION)')
+  #         echo "VERSION=$version" >> $GITHUB_ENV
+  #     - name: Create Executable (Linux-x64)
+  #       uses: sayyid5416/pyinstaller@v1
+  #       with:
+  #         python_ver: '3.11'
+  #         python_arch: 'x64'
+  #         spec: 'main.spec'
+  #         requirements: 'requirements.txt'
+  #         upload_exe_with_name: 'Zordon-${{env.VERSION}}-Linux-x64'
+  pyinstaller-build-macos:
+    runs-on: macos-latest
+    steps:
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+      # - name: Extract Version
+      #   id: get_version
+      #   run: |
+      #     version=$(python -c 'from version import APP_VERSION; print(APP_VERSION)')
+      #     echo "VERSION=$version" >> $GITHUB_ENV
+      - name: Create Executable (macOS-arm64)
+        uses: sayyid5416/pyinstaller@v1
+        with:
+          python_ver: '3.11'
+          python_arch: 'arm64'
+          spec: 'main.spec'
+          requirements: 'requirements.txt'
+          upload_exe_with_name: 'Zordon-testing-macOS-arm64'
+          options: --onefile

.github/workflows/pylint.yml

@@ -1,23 +0,0 @@
-name: Pylint
-
-on: [push]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install pylint
-    - name: Analysing the code with pylint
-      run: |
-        pylint $(git ls-files '*.py')

README.md

@@ -1,14 +1,37 @@
-# 🎬 Zordon - Render Management Tools
+# Zordon
 
-Welcome to Zordon! It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
+A tool designed for small render farms, such as those used in home studios or small businesses, to efficiently manage and run render jobs for Blender, FFMPEG, and other video renderers. It simplifies the process of distributing rendering tasks across multiple available machines, optimizing the rendering workflow for artists, animators, and video professionals.
 
-## 📦 Installation
+Notice: This should be considered a beta and is meant for casual / hobbiest use. Do not use in mission critical environments!
 
-Install the necessary dependencies: `pip3 install -r requirements.txt`
+## Supported Renderers
 
-## 🎨 Supported Renderers
-
-Zordon currently supports the following renderers:
+Zordon supports or plans to support the following renderers:
 
 - **Blender**
 - **FFMPEG**
+- **Adobe After Effects** ([coming soon](https://github.com/blw1138/Zordon/issues/84))
+- **Cinema 4D** ([planned](https://github.com/blw1138/Zordon/issues/105))
+- **Autodesk Maya** ([planned](https://github.com/blw1138/Zordon/issues/106))
+
+## System Requirements
+
+- Windows 10 or later
+- macOS Ventura (13.0) or later
+- Linux (Supported versions TBD)
+
+## Build using Pyinstaller
+
+Zordon is regularly tested with Python 3.11 and later. It's packaged and distributed with pyinstaller. It is supported on Windows, macOS and Linux.
+
+```
+git clone https://github.com/blw1138/Zordon.git
+pip3 install -r requirements.txt
+pip3 install pyinstaller
+pip3 install pyinstaller_versionfile
+pyinstaller main.spec
+```
+
+## License
+
+Zordon is licensed under the MIT License. See the [LICENSE](LICENSE.txt) file for more details.

main.spec

@@ -0,0 +1,126 @@
+# -*- mode: python ; coding: utf-8 -*-
+from PyInstaller.utils.hooks import collect_all
+
+# - get version from version file
+import os
+import sys
+import platform
+sys.path.insert(0, os.path.abspath('.'))
+from version import APP_NAME, APP_VERSION, APP_AUTHOR
+
+datas = [('resources', 'resources'), ('src/engines/blender/scripts/', 'src/engines/blender/scripts')]
+binaries = []
+hiddenimports = ['zeroconf']
+tmp_ret = collect_all('zeroconf')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+
+
+a = Analysis(
+    ['main.py'],
+    pathex=[],
+    binaries=binaries,
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+if platform.system() == 'Darwin':  # macOS
+
+    exe = EXE(
+        pyz,
+        a.scripts,
+        [],
+        exclude_binaries=True,
+        name=APP_NAME,
+        debug=False,
+        bootloader_ignore_signals=False,
+        strip=False,
+        upx=True,
+        console=False,
+        disable_windowed_traceback=False,
+        argv_emulation=False,
+        target_arch=None,
+        codesign_identity=None,
+        entitlements_file=None,
+    )
+    coll = COLLECT(
+        exe,
+        a.binaries,
+        a.datas,
+        strip=False,
+        upx=True,
+        upx_exclude=[],
+        name='server',
+    )
+    app = BUNDLE(
+        coll,
+        strip=True,
+        name=f'{APP_NAME}.app',
+        icon=None,
+        bundle_identifier=None,
+        version=APP_VERSION
+    )
+
+elif platform.system() == 'Windows':
+
+    import pyinstaller_versionfile
+    import tempfile
+
+    version_file_path = os.path.join(tempfile.gettempdir(), 'versionfile.txt')
+
+    pyinstaller_versionfile.create_versionfile(
+        output_file=version_file_path,
+        version=APP_VERSION,
+        company_name=APP_AUTHOR,
+        file_description=APP_NAME,
+        internal_name=APP_NAME,
+        legal_copyright=f"© {APP_AUTHOR}",
+        original_filename=f"{APP_NAME}.exe",
+        product_name=APP_NAME
+    )
+
+    exe = EXE(
+        pyz,
+        a.scripts,
+        a.binaries,
+        a.datas,
+        [],
+        name=APP_NAME,
+        debug=False,
+        bootloader_ignore_signals=False,
+        strip=False,
+        upx=True,
+        console=False,
+        disable_windowed_traceback=False,
+        argv_emulation=False,
+        target_arch=None,
+        codesign_identity=None,
+        entitlements_file=None,
+        version=version_file_path
+    )
+
+else:  # linux
+    exe = EXE(
+        pyz,
+        a.scripts,
+        a.binaries,
+        a.datas,
+        [],
+        name=APP_NAME,
+        debug=False,
+        bootloader_ignore_signals=False,
+        strip=False,
+        upx=True,
+        console=False,
+        disable_windowed_traceback=False,
+        argv_emulation=False,
+        target_arch=None,
+        codesign_identity=None,
+        entitlements_file=None
+    )

requirements.txt

@@ -34,4 +34,5 @@ kiwisolver>=1.4.5
 attrs>=23.2.0
 lxml>=5.1.0
 click>=8.1.7
-requests_toolbelt>=1.0.0
+requests_toolbelt>=1.0.0
+pyinstaller_versionfile>=2.1.1

setup.py

@@ -1,22 +0,0 @@
-"""
-This is a setup.py script generated by py2applet
-
-Usage:
-    python setup.py py2app
-"""
-import glob
-
-from setuptools import setup
-
-APP = ['main.py']
-DATA_FILES = [('config', glob.glob('config/*.*')),
-              ('resources', glob.glob('resources/*.*'))]
-OPTIONS = {}
-
-setup(
-    app=APP,
-    data_files=DATA_FILES,
-    options={'py2app': OPTIONS},
-    setup_requires=['py2app'],
-    name='Zordon'
-)

src/api/api_server.py

@@ -10,143 +10,84 @@ import ssl
 import tempfile
 import time
 from datetime import datetime
-from zipfile import ZipFile
 
 import psutil
 import yaml
-from flask import Flask, request, send_file, after_this_request, Response, redirect, url_for, abort
+from flask import Flask, request, send_file, after_this_request, Response, redirect, url_for
 from sqlalchemy.orm.exc import DetachedInstanceError
 
 from src.api.add_job_helpers import handle_uploaded_project_files, process_zipped_project
 from src.api.preview_manager import PreviewManager
 from src.distributed_job_manager import DistributedJobManager
-from src.engines.core.base_worker import string_to_status, RenderStatus
 from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue, JobNotFoundError
-from src.utilities.benchmark import cpu_benchmark, disk_io_benchmark
 from src.utilities.config import Config
 from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, \
     current_system_os_version, num_to_alphanumeric
-from src.utilities.zeroconf_server import ZeroconfServer
+from src.utilities.status_utils import string_to_status
 
 logger = logging.getLogger()
 server = Flask(__name__)
 ssl._create_default_https_context = ssl._create_unverified_context  # disable SSL for downloads
 
-categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
-              RenderStatus.COMPLETED, RenderStatus.CANCELLED]
+
+def start_server(hostname=None):
+
+    # get hostname
+    if not hostname:
+        local_hostname = socket.gethostname()
+        hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
+
+    # load flask settings
+    server.config['HOSTNAME'] = hostname
+    server.config['PORT'] = int(Config.port_number)
+    server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(Config.upload_folder))
+    server.config['MAX_CONTENT_PATH'] = Config.max_content_path
+    server.config['enable_split_jobs'] = Config.enable_split_jobs
+
+    # disable most Flask logging
+    flask_log = logging.getLogger('werkzeug')
+    flask_log.setLevel(Config.flask_log_level.upper())
+
+    logger.debug('Starting API server')
+    server.run(host='0.0.0.0', port=server.config['PORT'], debug=Config.flask_debug_enable, use_reloader=False,
+               threaded=True)
 
 
-# -- Error Handlers --
-
-@server.errorhandler(JobNotFoundError)
-def handle_job_not_found(job_error):
-    return str(job_error), 400
-
-
-@server.errorhandler(DetachedInstanceError)
-def handle_detached_instance(error):
-    # logger.debug(f"detached instance: {error}")
-    return "Unavailable", 503
-
-
-@server.errorhandler(Exception)
-def handle_general_error(general_error):
-    err_msg = f"Server error: {general_error}"
-    logger.error(err_msg)
-    return err_msg, 500
-
-
-# -- Jobs --
-
-
-def sorted_jobs(all_jobs, sort_by_date=True):
-    if not sort_by_date:
-        sorted_job_list = []
-        if all_jobs:
-            for status_category in categories:
-                found_jobs = [x for x in all_jobs if x.status == status_category.value]
-                if found_jobs:
-                    sorted_found_jobs = sorted(found_jobs, key=lambda d: d.date_created, reverse=True)
-                    sorted_job_list.extend(sorted_found_jobs)
-    else:
-        sorted_job_list = sorted(all_jobs, key=lambda d: d.date_created, reverse=True)
-    return sorted_job_list
-
+# --------------------------------------------
+# Get All Jobs
+# --------------------------------------------
 
 @server.get('/api/jobs')
 def jobs_json():
-    try:
-        all_jobs = [x.json() for x in RenderQueue.all_jobs()]
-        job_cache_int = int(json.dumps(all_jobs).__hash__())
-        job_cache_token = num_to_alphanumeric(job_cache_int)
-        return {'jobs': all_jobs, 'token': job_cache_token}
-    except DetachedInstanceError as e:
-        raise e
-    except Exception as e:
-        logger.error(f"Error fetching jobs_json: {e}")
-        raise e
+    """Retrieves all jobs from the render queue in JSON format.
+
+    This endpoint fetches all jobs currently in the render queue, converts them to JSON format,
+    and returns them along with a cache token that represents the current state of the job list.
+
+    Returns:
+        dict: A dictionary containing:
+            - 'jobs' (list[dict]): A list of job dictionaries, each representing a job in the queue.
+            - 'token' (str): A cache token generated from the hash of the job list.
+    """
+    all_jobs = [x.json() for x in RenderQueue.all_jobs()]
+    job_cache_int = int(json.dumps(all_jobs).__hash__())
+    job_cache_token = num_to_alphanumeric(job_cache_int)
+    return {'jobs': all_jobs, 'token': job_cache_token}
 
 
 @server.get('/api/jobs_long_poll')
 def long_polling_jobs():
-    try:
-        hash_token = request.args.get('token', None)
-        start_time = time.time()
-        while True:
-            all_jobs = jobs_json()
-            if all_jobs['token'] != hash_token:
-                return all_jobs
-            # Break after 30 seconds to avoid gateway timeout
-            if time.time() - start_time > 30:
-                return {}, 204
-            time.sleep(1)
-    except DetachedInstanceError as e:
-        raise e
-    except Exception as e:
-        logger.error(f"Error fetching long_polling_jobs: {e}")
-        raise e
-
-
-@server.route('/api/job/<job_id>/thumbnail')
-def job_thumbnail(job_id):
-
-    try:
-        big_thumb = request.args.get('size', False) == "big"
-        video_ok = request.args.get('video_ok', False)
-        found_job = RenderQueue.job_with_id(job_id, none_ok=False)
-
-        # trigger a thumbnail update - just in case
-        PreviewManager.update_previews_for_job(found_job, wait_until_completion=True, timeout=60)
-        previews = PreviewManager.get_previews_for_job(found_job)
-        all_previews_list = previews.get('output', previews.get('input', []))
-
-        video_previews = [x for x in all_previews_list if x['kind'] == 'video']
-        image_previews = [x for x in all_previews_list if x['kind'] == 'image']
-        filtered_list = video_previews if video_previews and video_ok else image_previews
-
-        # todo - sort by size or other metrics here
-        if filtered_list:
-            preview_to_send = filtered_list[0]
-            mime_types = {'image': 'image/jpeg', 'video': 'video/mp4'}
-            file_mime_type = mime_types.get(preview_to_send['kind'], 'unknown')
-            return send_file(preview_to_send['filename'], mimetype=file_mime_type)
-    except Exception as e:
-        logger.error(f'Error getting thumbnail: {e}')
-        return f'Error getting thumbnail: {e}', 500
-    return "No thumbnail available", 404
-
-
-# Get job file routing
-@server.route('/api/job/<job_id>/file/<filename>', methods=['GET'])
-def get_job_file(job_id, filename):
-    found_job = RenderQueue.job_with_id(job_id)
-    try:
-        for full_path in found_job.file_list():
-            if filename in full_path:
-                return send_file(path_or_file=full_path)
-    except FileNotFoundError:
-        abort(404)
+    hash_token = request.args.get('token', None)
+    start_time = time.time()
+    while True:
+        all_jobs = jobs_json()
+        if all_jobs['token'] != hash_token:
+            return all_jobs
+        # Break after 30 seconds to avoid gateway timeout
+        if time.time() - start_time > 30:
+            return {}, 204
+        time.sleep(1)
 
 
 @server.get('/api/jobs/<status_val>')
@@ -159,24 +100,33 @@ def filtered_jobs_json(status_val):
         return f'Cannot find jobs with status {status_val}', 400
 
 
-@server.post('/api/job/<job_id>/send_subjob_update_notification')
-def subjob_update_notification(job_id):
-    try:
-        subjob_details = request.json
-        logger.info(f"Subjob to job id: {job_id} is now {subjob_details['status']}")
-        DistributedJobManager.handle_subjob_update_notification(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
-        return Response(status=200)
-    except JobNotFoundError:
-        return "Job not found", 404
-
+# --------------------------------------------
+# Job Details / File Handling
+# --------------------------------------------
 
 @server.get('/api/job/<job_id>')
-def get_job_status(job_id):
+def get_job_details(job_id):
+    """Retrieves the details of a requested job in JSON format
+
+    Args:
+        job_id (str): The ID of the render job.
+
+    Returns:
+        dict: A JSON representation of the job's details.
+    """
     return RenderQueue.job_with_id(job_id).json()
 
 
 @server.get('/api/job/<job_id>/logs')
 def get_job_logs(job_id):
+    """Retrieves the log file for a specific render job.
+
+    Args:
+        job_id (str): The ID of the render job.
+
+    Returns:
+        Response: The log file's content as plain text, or an empty response if the log file is not found.
+    """
     found_job = RenderQueue.job_with_id(job_id)
     log_path = system_safe_path(found_job.log_path())
     log_data = None
@@ -192,7 +142,7 @@ def get_file_list(job_id):
 
 
 @server.route('/api/job/<job_id>/download')
-def download_file(job_id):
+def download_requested_file(job_id):
 
     requested_filename = request.args.get('filename')
     if not requested_filename:
@@ -207,7 +157,7 @@ def download_file(job_id):
 
 
 @server.route('/api/job/<job_id>/download_all')
-def download_all(job_id):
+def download_all_files(job_id):
     zip_filename = None
 
     @after_this_request
@@ -222,6 +172,7 @@ def download_all(job_id):
     found_job = RenderQueue.job_with_id(job_id)
     output_dir = os.path.dirname(found_job.output_path)
     if os.path.exists(output_dir):
+        from zipfile import ZipFile
         zip_filename = system_safe_path(os.path.join(tempfile.gettempdir(),
                                                      pathlib.Path(found_job.input_path).stem + '.zip'))
         with ZipFile(zip_filename, 'w') as zipObj:
@@ -233,6 +184,10 @@ def download_all(job_id):
         return f'Cannot find project files for job {job_id}', 500
 
 
+# --------------------------------------------
+# System Environment / Status
+# --------------------------------------------
+
 @server.get('/api/presets')
 def presets():
     presets_path = system_safe_path('config/presets.yaml')
@@ -264,13 +219,28 @@ def snapshot():
     return server_data
 
 
-@server.get('/api/_detected_clients')
-def detected_clients():
-    # todo: dev/debug only. Should not ship this - probably.
-    return ZeroconfServer.found_hostnames()
+@server.route('/api/status')
+def status():
+    return {"timestamp": datetime.now().isoformat(),
+            "system_os": current_system_os(),
+            "system_os_version": current_system_os_version(),
+            "system_cpu": current_system_cpu(),
+            "cpu_percent": psutil.cpu_percent(percpu=False),
+            "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
+            "cpu_count": psutil.cpu_count(logical=False),
+            "memory_total": psutil.virtual_memory().total,
+            "memory_available": psutil.virtual_memory().available,
+            "memory_percent": psutil.virtual_memory().percent,
+            "job_counts": RenderQueue.job_counts(),
+            "hostname": server.config['HOSTNAME'],
+            "port": server.config['PORT']
+            }
 
 
-# New version
+# --------------------------------------------
+# Job Lifecyle (Create, Cancel, Delete)
+# --------------------------------------------
+
 @server.post('/api/add_job')
 def add_job_handler():
     # Process request data
@@ -324,100 +294,84 @@ def delete_job(job_id):
 
         # Check if we can remove the 'output' directory
         found_job = RenderQueue.job_with_id(job_id)
+        project_dir = os.path.dirname(os.path.dirname(found_job.input_path))
         output_dir = os.path.dirname(found_job.output_path)
-        if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
-            shutil.rmtree(output_dir)
+        found_job.stop()
 
         try:
             PreviewManager.delete_previews_for_job(found_job)
         except Exception as e:
             logger.error(f"Error deleting previews for {found_job}: {e}")
 
-        # See if we own the project_dir (i.e. was it uploaded)
-        project_dir = os.path.dirname(os.path.dirname(found_job.input_path))
-        if server.config['UPLOAD_FOLDER'] in project_dir and os.path.exists(project_dir):
-            # check to see if any other projects are sharing the same project file
-            project_dir_files = [f for f in os.listdir(project_dir) if not f.startswith('.')]
-            if len(project_dir_files) == 0 or (len(project_dir_files) == 1 and 'source' in project_dir_files[0]):
-                logger.info(f"Removing project directory: {project_dir}")
-                shutil.rmtree(project_dir)
-
+        # finally delete the job
         RenderQueue.delete_job(found_job)
-        if request.args.get('redirect', False):
-            return redirect(url_for('index'))
-        else:
-            return "Job deleted", 200
 
+        # delete the output_dir
+        if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
+            shutil.rmtree(output_dir)
+
+        # See if we own the project_dir (i.e. was it uploaded) - if so delete the directory
+        try:
+            if server.config['UPLOAD_FOLDER'] in project_dir and os.path.exists(project_dir):
+                # check to see if any other projects are sharing the same project file
+                project_dir_files = [f for f in os.listdir(project_dir) if not f.startswith('.')]
+                if len(project_dir_files) == 0 or (len(project_dir_files) == 1 and 'source' in project_dir_files[0]):
+                    logger.info(f"Removing project directory: {project_dir}")
+                    shutil.rmtree(project_dir)
+        except Exception as e:
+            logger.error(f"Error removing project files: {e}")
+
+        return "Job deleted", 200
     except Exception as e:
         logger.error(f"Error deleting job: {e}")
         return f"Error deleting job: {e}", 500
 
 
-@server.get('/api/clear_history')
-def clear_history():
-    RenderQueue.clear_history()
-    return 'success'
-
-
-@server.route('/api/status')
-def status():
-
-    # Get system info
-    return {"timestamp": datetime.now().isoformat(),
-            "system_os": current_system_os(),
-            "system_os_version": current_system_os_version(),
-            "system_cpu": current_system_cpu(),
-            "cpu_percent": psutil.cpu_percent(percpu=False),
-            "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
-            "cpu_count": psutil.cpu_count(logical=False),
-            "memory_total": psutil.virtual_memory().total,
-            "memory_available": psutil.virtual_memory().available,
-            "memory_percent": psutil.virtual_memory().percent,
-            "job_counts": RenderQueue.job_counts(),
-            "hostname": server.config['HOSTNAME'],
-            "port": server.config['PORT']
-            }
-
+# --------------------------------------------
+# Engine Info and Management:
+# --------------------------------------------
 
 @server.get('/api/renderer_info')
 def renderer_info():
-
     response_type = request.args.get('response_type', 'standard')
+    if response_type not in ['full', 'standard']:
+        raise ValueError(f"Invalid response_type: {response_type}")
 
     def process_engine(engine):
         try:
             # Get all installed versions of the engine
             installed_versions = EngineManager.all_versions_for_engine(engine.name())
-            if installed_versions:
-                # Use system-installed versions to avoid permission issues
-                system_installed_versions = [x for x in installed_versions if x['type'] == 'system']
-                install_path = system_installed_versions[0]['path'] if system_installed_versions else \
-                installed_versions[0]['path']
+            if not installed_versions:
+                return None
 
-                en = engine(install_path)
+            system_installed_versions = [v for v in installed_versions if v['type'] == 'system']
+            install_path = system_installed_versions[0]['path'] if system_installed_versions else installed_versions[0]['path']
 
-                if response_type == 'full':  # Full dataset - Can be slow
-                    return {
-                        en.name(): {
-                            'is_available': RenderQueue.is_available_for_job(en.name()),
-                            'versions': installed_versions,
-                            'supported_extensions': engine.supported_extensions(),
-                            'supported_export_formats': en.get_output_formats(),
-                            'system_info': en.system_info()
-                        }
+            en = engine(install_path)
+            engine_name = en.name()
+            result = {
+                engine_name: {
+                    'is_available': RenderQueue.is_available_for_job(engine_name),
+                    'versions': installed_versions
+                }
+            }
+
+            if response_type == 'full':
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future_results = {
+                        'supported_extensions': executor.submit(en.supported_extensions),
+                        'supported_export_formats': executor.submit(en.get_output_formats),
+                        'system_info': executor.submit(en.system_info)
                     }
-                elif response_type == 'standard':  # Simpler dataset to reduce response times
-                    return {
-                        en.name(): {
-                            'is_available': RenderQueue.is_available_for_job(en.name()),
-                            'versions': installed_versions,
-                        }
-                    }
-                else:
-                    raise AttributeError(f"Invalid response_type: {response_type}")
+
+                    for key, future in future_results.items():
+                        result[engine_name][key] = future.result()
+
+            return result
+
         except Exception as e:
             logger.error(f'Error fetching details for {engine.name()} renderer: {e}')
-            return {}
+            raise e
 
     renderer_data = {}
     with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -497,35 +451,95 @@ def get_renderer_help(renderer):
         return f"Cannot find renderer '{renderer}'", 400
 
 
+# --------------------------------------------
+# Miscellaneous:
+# --------------------------------------------
+@server.post('/api/job/<job_id>/send_subjob_update_notification')
+def subjob_update_notification(job_id):
+    subjob_details = request.json
+    DistributedJobManager.handle_subjob_update_notification(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
+    return Response(status=200)
+
+
+@server.route('/api/job/<job_id>/thumbnail')
+def job_thumbnail(job_id):
+
+    try:
+        big_thumb = request.args.get('size', False) == "big"
+        video_ok = request.args.get('video_ok', False)
+        found_job = RenderQueue.job_with_id(job_id, none_ok=False)
+
+        # trigger a thumbnail update - just in case
+        PreviewManager.update_previews_for_job(found_job, wait_until_completion=True, timeout=60)
+        previews = PreviewManager.get_previews_for_job(found_job)
+        all_previews_list = previews.get('output', previews.get('input', []))
+
+        video_previews = [x for x in all_previews_list if x['kind'] == 'video']
+        image_previews = [x for x in all_previews_list if x['kind'] == 'image']
+        filtered_list = video_previews if video_previews and video_ok else image_previews
+
+        # todo - sort by size or other metrics here
+        if filtered_list:
+            preview_to_send = filtered_list[0]
+            mime_types = {'image': 'image/jpeg', 'video': 'video/mp4'}
+            file_mime_type = mime_types.get(preview_to_send['kind'], 'unknown')
+            return send_file(preview_to_send['filename'], mimetype=file_mime_type)
+    except Exception as e:
+        logger.error(f'Error getting thumbnail: {e}')
+        return f'Error getting thumbnail: {e}', 500
+    return "No thumbnail available", 404
+
+
+# --------------------------------------------
+# System Benchmarks:
+# --------------------------------------------
+
 @server.get('/api/cpu_benchmark')
 def get_cpu_benchmark_score():
+    from src.utilities.benchmark import cpu_benchmark
     return str(cpu_benchmark(10))
 
 
 @server.get('/api/disk_benchmark')
 def get_disk_benchmark():
+    from src.utilities.benchmark import disk_io_benchmark
     results = disk_io_benchmark()
     return {'write_speed': results[0], 'read_speed': results[-1]}
 
 
-def start_server(hostname=None):
+# --------------------------------------------
+# Error Handlers:
+# --------------------------------------------
 
-    # get hostname
-    if not hostname:
-        local_hostname = socket.gethostname()
-        hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
+@server.errorhandler(JobNotFoundError)
+def handle_job_not_found(job_error):
+    return str(job_error), 400
 
-    # load flask settings
-    server.config['HOSTNAME'] = hostname
-    server.config['PORT'] = int(Config.port_number)
-    server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(Config.upload_folder))
-    server.config['MAX_CONTENT_PATH'] = Config.max_content_path
-    server.config['enable_split_jobs'] = Config.enable_split_jobs
 
-    # disable most Flask logging
-    flask_log = logging.getLogger('werkzeug')
-    flask_log.setLevel(Config.flask_log_level.upper())
+@server.errorhandler(DetachedInstanceError)
+def handle_detached_instance(_):
+    return "Unavailable", 503
 
-    logger.debug('Starting API server')
-    server.run(host='0.0.0.0', port=server.config['PORT'], debug=Config.flask_debug_enable, use_reloader=False,
-               threaded=True)
+
+@server.errorhandler(Exception)
+def handle_general_error(general_error):
+    err_msg = f"Server error: {general_error}"
+    logger.error(err_msg)
+    return err_msg, 500
+
+
+# --------------------------------------------
+# Debug / Development Only:
+# --------------------------------------------
+
+@server.get('/api/_debug/detected_clients')
+def detected_clients():
+    # todo: dev/debug only. Should not ship this - probably.
+    from src.utilities.zeroconf_server import ZeroconfServer
+    return ZeroconfServer.found_hostnames()
+
+
+@server.get('/api/_debug/clear_history')
+def clear_history():
+    RenderQueue.clear_history()
+    return 'success'

src/api/preview_manager.py

@@ -7,8 +7,8 @@ from pathlib import Path
 from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
 
 logger = logging.getLogger()
-supported_video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', 'mkv']
-supported_image_formats = ['.jpg', '.png', '.exr', '.tif']
+supported_video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', '.mkv', '.webm']
+supported_image_formats = ['.jpg', '.png', '.exr', '.tif', '.tga', '.bmp', '.webp']
 
 
 class PreviewManager:

src/api/server_proxy.py

@@ -10,7 +10,6 @@ from urllib.parse import urljoin
 
 from src.utilities.misc_helper import is_localhost
 from src.utilities.status_utils import RenderStatus
-from src.utilities.zeroconf_server import ZeroconfServer
 
 status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green',
                  RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple',
@@ -26,15 +25,8 @@ LOOPBACK = '127.0.0.1'
 
 
 class RenderServerProxy:
-    """
-    The ServerProxy class is responsible for interacting with a remote server.
-    It provides methods to request data from the server and store the status of the server.
-
-    Attributes:
-        system_cpu (str): The CPU type of the system.
-        system_cpu_count (int): The number of CPUs in the system.
-        system_os (str): The operating system of the system.
-        system_os_version (str): The version of the operating system.
+    """The ServerProxy class is responsible for interacting with a remote server.
+    It provides convenience methods to request data from the server and store the status of the server.
     """
     
     def __init__(self, hostname, server_port="8080"):
@@ -55,6 +47,10 @@ class RenderServerProxy:
         self.system_os = None
         self.system_os_version = None
 
+    # --------------------------------------------
+    # Basics / Connection:
+    # --------------------------------------------
+
     def __repr__(self):
         return f"<RenderServerProxy - {self.hostname}>"
 
@@ -73,6 +69,10 @@ class RenderServerProxy:
         running_jobs = [x for x in self.__jobs_cache if x['status'] == 'running'] if self.__jobs_cache else []
         return f"{len(running_jobs)} running" if running_jobs else "Ready"
 
+    # --------------------------------------------
+    # Requests:
+    # --------------------------------------------
+
     def request_data(self, payload, timeout=5):
         try:
             req = self.request(payload, timeout)
@@ -103,6 +103,10 @@ class RenderServerProxy:
         hostname = LOOPBACK if self.is_localhost else self.hostname
         return requests.get(f'http://{hostname}:{self.port}/api/{payload}', timeout=timeout)
 
+    # --------------------------------------------
+    # Background Updates:
+    # --------------------------------------------
+
     def start_background_update(self):
         if self.__update_in_background:
             return
@@ -119,17 +123,6 @@ class RenderServerProxy:
         self.__background_thread.daemon = True
         self.__background_thread.start()
 
-    def stop_background_update(self):
-        self.__update_in_background = False
-
-    def get_job_info(self, job_id, timeout=5):
-        return self.request_data(f'job/{job_id}', timeout=timeout)
-
-    def get_all_jobs(self, timeout=5, ignore_token=False):
-        if not self.__update_in_background or ignore_token:
-            self.__update_job_cache(timeout, ignore_token)
-        return self.__jobs_cache.copy() if self.__jobs_cache else None
-
     def __update_job_cache(self, timeout=40, ignore_token=False):
 
         if self.__offline_flags:  # if we're offline, don't bother with the long poll
@@ -147,15 +140,21 @@ class RenderServerProxy:
             self.__jobs_cache = sorted_jobs
             self.__jobs_cache_token = status_result['token']
 
+    def stop_background_update(self):
+        self.__update_in_background = False
+
+    # --------------------------------------------
+    # Get System Info:
+    # --------------------------------------------
+
+    def get_all_jobs(self, timeout=5, ignore_token=False):
+        if not self.__update_in_background or ignore_token:
+            self.__update_job_cache(timeout, ignore_token)
+        return self.__jobs_cache.copy() if self.__jobs_cache else None
+
     def get_data(self, timeout=5):
         return self.request_data('full_status', timeout=timeout)
 
-    def cancel_job(self, job_id, confirm=False):
-        return self.request_data(f'job/{job_id}/cancel?confirm={confirm}')
-
-    def delete_job(self, job_id, confirm=False):
-        return self.request_data(f'job/{job_id}/delete?confirm={confirm}')
-
     def get_status(self):
         status = self.request_data('status')
         if status and not self.system_cpu:
@@ -165,26 +164,19 @@ class RenderServerProxy:
             self.system_os_version = status['system_os_version']
         return status
 
-    def is_engine_available(self, engine_name):
-        return self.request_data(f'{engine_name}/is_available')
+    # --------------------------------------------
+    # Get Job Info:
+    # --------------------------------------------
 
-    def get_all_engines(self):
-        return self.request_data('all_engines')
+    def get_job_info(self, job_id, timeout=5):
+        return self.request_data(f'job/{job_id}', timeout=timeout)
 
-    def send_subjob_update_notification(self, parent_id, subjob):
-        """
-        Notifies the parent job of an update in a subjob.
+    def get_job_files_list(self, job_id):
+        return self.request_data(f"job/{job_id}/file_list")
 
-        Args:
-            parent_id (str): The ID of the parent job.
-            subjob (Job): The subjob that has updated.
-
-        Returns:
-            Response: The response from the server.
-        """
-        hostname = LOOPBACK if self.is_localhost else self.hostname
-        return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
-                             json=subjob.json())
+    # --------------------------------------------
+    # Job Lifecycle:
+    # --------------------------------------------
 
     def post_job_to_server(self, file_path, job_list, callback=None):
         """
@@ -232,29 +224,36 @@ class RenderServerProxy:
         except Exception as e:
             logger.error(f"An error occurred: {e}")
 
-    def get_job_files_list(self, job_id):
-        return self.request_data(f"job/{job_id}/file_list")
+    def cancel_job(self, job_id, confirm=False):
+        return self.request_data(f'job/{job_id}/cancel?confirm={confirm}')
 
-    def download_all_job_files(self, job_id, save_path):
+    def delete_job(self, job_id, confirm=False):
+        return self.request_data(f'job/{job_id}/delete?confirm={confirm}')
+
+    def send_subjob_update_notification(self, parent_id, subjob):
+        """
+        Notifies the parent job of an update in a subjob.
+
+        Args:
+            parent_id (str): The ID of the parent job.
+            subjob (Job): The subjob that has updated.
+
+        Returns:
+            Response: The response from the server.
+        """
         hostname = LOOPBACK if self.is_localhost else self.hostname
-        url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
-        return self.__download_file_from_url(url, output_filepath=save_path)
+        return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
+                             json=subjob.json())
 
-    def download_job_file(self, job_id, job_filename, save_path):
-        hostname = LOOPBACK if self.is_localhost else self.hostname
-        url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
-        return self.__download_file_from_url(url, output_filepath=save_path)
+    # --------------------------------------------
+    # Renderers:
+    # --------------------------------------------
 
-    @staticmethod
-    def __download_file_from_url(url, output_filepath):
-        with requests.get(url, stream=True) as r:
-            r.raise_for_status()
-            with open(output_filepath, 'wb') as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-        return output_filepath
+    def is_engine_available(self, engine_name):
+        return self.request_data(f'{engine_name}/is_available')
 
-    # --- Renderer --- #
+    def get_all_engines(self):
+        return self.request_data('all_engines')
 
     def get_renderer_info(self, response_type='standard', timeout=5):
         """
@@ -285,3 +284,26 @@ class RenderServerProxy:
         form_data = {'engine': engine, 'version': version, 'system_cpu': system_cpu}
         hostname = LOOPBACK if self.is_localhost else self.hostname
         return requests.post(f'http://{hostname}:{self.port}/api/delete_engine', json=form_data)
+
+    # --------------------------------------------
+    # Download Files:
+    # --------------------------------------------
+
+    def download_all_job_files(self, job_id, save_path):
+        hostname = LOOPBACK if self.is_localhost else self.hostname
+        url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
+        return self.__download_file_from_url(url, output_filepath=save_path)
+
+    def download_job_file(self, job_id, job_filename, save_path):
+        hostname = LOOPBACK if self.is_localhost else self.hostname
+        url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
+        return self.__download_file_from_url(url, output_filepath=save_path)
+
+    @staticmethod
+    def __download_file_from_url(url, output_filepath):
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+            with open(output_filepath, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        return output_filepath

src/distributed_job_manager.py

@@ -3,10 +3,7 @@ import os
 import socket
 import threading
 import time
-import zipfile
-from concurrent.futures import ThreadPoolExecutor
 
-import requests
 from plyer import notification
 from pubsub import pub
 
@@ -15,7 +12,7 @@ from src.api.server_proxy import RenderServerProxy
 from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue
 from src.utilities.config import Config
-from src.utilities.misc_helper import get_file_size_human
+from src.utilities.server_helper import download_missing_frames_from_subjob, distribute_server_work
 from src.utilities.status_utils import RenderStatus, string_to_status
 from src.utilities.zeroconf_server import ZeroconfServer
 
@@ -42,10 +39,10 @@ class DistributedJobManager:
         """
         Responds to the 'frame_complete' pubsub message for local jobs.
 
-        Parameters:
-        job_id (str): The ID of the job that has changed status.
-        old_status (str): The previous status of the job.
-        new_status (str): The new (current) status of the job.
+        Args:
+            job_id (str): The ID of the job that has changed status.
+            old_status (str): The previous status of the job.
+            new_status (str): The new (current) status of the job.
 
         Note: Do not call directly. Instead, call via the 'frame_complete' pubsub message.
         """
@@ -78,10 +75,10 @@ class DistributedJobManager:
         Responds to the 'status_change' pubsub message for local jobs.
         If it's a child job, it notifies the parent job about the status change.
 
-        Parameters:
-        job_id (str): The ID of the job that has changed status.
-        old_status (str): The previous status of the job.
-        new_status (str): The new (current) status of the job.
+        Args:
+            job_id (str): The ID of the job that has changed status.
+            old_status (str): The previous status of the job.
+            new_status (str): The new (current) status of the job.
 
         Note: Do not call directly. Instead, call via the 'status_change' pubsub message.
         """
@@ -132,14 +129,12 @@ class DistributedJobManager:
     # --------------------------------------------
 
     @classmethod
-    def create_render_job(cls, job_data, loaded_project_local_path):
-        """
-        Creates render jobs.
-
-        This method  job data and a local path to a loaded project. It creates and returns new a render job.
+    def create_render_job(cls, new_job_attributes, loaded_project_local_path):
+        """Creates render jobs. Pass in dict of job_data and the local path to the project. It creates and returns a new
+        render job.
 
         Args:
-            job_data (dict): Job data.
+            new_job_attributes (dict): Dict of desired attributes for new job (frame count, renderer, output path, etc)
             loaded_project_local_path (str): The local path to the loaded project.
 
         Returns:
@@ -147,7 +142,7 @@ class DistributedJobManager:
         """
 
         # get new output path in output_dir
-        output_path = job_data.get('output_path')
+        output_path = new_job_attributes.get('output_path')
         if not output_path:
             loaded_project_filename = os.path.basename(loaded_project_local_path)
             output_filename = os.path.splitext(loaded_project_filename)[0]
@@ -161,27 +156,27 @@ class DistributedJobManager:
         logger.debug(f"New job output path: {output_path}")
 
         # create & configure jobs
-        worker = EngineManager.create_worker(renderer=job_data['renderer'],
+        worker = EngineManager.create_worker(renderer=new_job_attributes['renderer'],
                                              input_path=loaded_project_local_path,
                                              output_path=output_path,
-                                             engine_version=job_data.get('engine_version'),
-                                             args=job_data.get('args', {}),
-                                             parent=job_data.get('parent'),
-                                             name=job_data.get('name'))
-        worker.status = job_data.get("initial_status", worker.status)  # todo: is this necessary?
-        worker.priority = int(job_data.get('priority', worker.priority))
-        worker.start_frame = int(job_data.get("start_frame", worker.start_frame))
-        worker.end_frame = int(job_data.get("end_frame", worker.end_frame))
+                                             engine_version=new_job_attributes.get('engine_version'),
+                                             args=new_job_attributes.get('args', {}),
+                                             parent=new_job_attributes.get('parent'),
+                                             name=new_job_attributes.get('name'))
+        worker.status = new_job_attributes.get("initial_status", worker.status)  # todo: is this necessary?
+        worker.priority = int(new_job_attributes.get('priority', worker.priority))
+        worker.start_frame = int(new_job_attributes.get("start_frame", worker.start_frame))
+        worker.end_frame = int(new_job_attributes.get("end_frame", worker.end_frame))
         worker.watchdog_timeout = Config.worker_process_timeout
         worker.hostname = socket.gethostname()
 
         # determine if we can / should split the job
-        if job_data.get("enable_split_jobs", False) and (worker.total_frames > 1) and not worker.parent:
-            cls.split_into_subjobs_async(worker, job_data, loaded_project_local_path)
+        if new_job_attributes.get("enable_split_jobs", False) and (worker.total_frames > 1) and not worker.parent:
+            cls.split_into_subjobs_async(worker, new_job_attributes, loaded_project_local_path)
         else:
-            logger.debug("Not splitting into subjobs")
+            worker.status = RenderStatus.NOT_STARTED
 
-        RenderQueue.add_to_render_queue(worker, force_start=job_data.get('force_start', False))
+        RenderQueue.add_to_render_queue(worker, force_start=new_job_attributes.get('force_start', False))
         PreviewManager.update_previews_for_job(worker)
 
         return worker
@@ -192,8 +187,7 @@ class DistributedJobManager:
 
     @classmethod
     def handle_subjob_update_notification(cls, local_job, subjob_data):
-        """
-        Responds to a notification from a remote subjob and the host requests any subsequent updates from the subjob.
+        """Responds to a notification from a remote subjob and the host requests any subsequent updates from the subjob.
 
         Args:
             local_job (BaseRenderWorker): The local parent job worker.
@@ -211,145 +205,95 @@ class DistributedJobManager:
         if old_status != subjob_status.value:
             logger.debug(f"Subjob status changed: {logname} -> {subjob_status.value}")
 
-        cls.download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
-
-    @staticmethod
-    def download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname):
-
-        try:
-            local_files = [os.path.basename(x) for x in local_job.file_list()]
-            subjob_proxy = RenderServerProxy(subjob_hostname)
-            subjob_files = subjob_proxy.get_job_files_list(job_id=subjob_id) or []
-
-            for subjob_filename in subjob_files:
-                if subjob_filename not in local_files:
-                    try:
-                        logger.debug(f"Downloading new file '{subjob_filename}' from {subjob_hostname}")
-                        local_save_path = os.path.join(os.path.dirname(local_job.output_path), subjob_filename)
-                        subjob_proxy.download_job_file(job_id=subjob_id, job_filename=subjob_filename,
-                                                       save_path=local_save_path)
-                        logger.debug(f'Downloaded successfully - {local_save_path}')
-                    except Exception as e:
-                        logger.error(f"Error downloading file '{subjob_filename}' from {subjob_hostname}: {e}")
-        except Exception as e:
-            logger.exception(f'Uncaught exception while trying to download from subjob: {e}')
-
-    @staticmethod
-    def download_all_from_subjob(local_job, subjob_id, subjob_hostname):
-        """
-        Downloads and extracts files from a completed subjob on a remote server.
-
-        Parameters:
-        local_job (BaseRenderWorker): The local parent job worker.
-        subjob_id (str or int): The ID of the subjob.
-        subjob_hostname (str): The hostname of the remote server where the subjob is located.
-
-        Returns:
-        bool: True if the files have been downloaded and extracted successfully, False otherwise.
-        """
-
-        child_key = f'{subjob_id}@{subjob_hostname}'
-        logname = f"{local_job.id}:{child_key}"
-        zip_file_path = local_job.output_path + f'_{subjob_hostname}_{subjob_id}.zip'
-
-        # download zip file from server
-        try:
-            local_job.children[child_key]['download_status'] = 'working'
-            logger.info(f"Downloading completed subjob files from {subjob_hostname} to localhost")
-            RenderServerProxy(subjob_hostname).download_all_job_files(subjob_id, zip_file_path)
-            logger.info(f"File transfer complete for {logname} - Transferred {get_file_size_human(zip_file_path)}")
-        except Exception as e:
-            logger.error(f"Error downloading files from remote server: {e}")
-            local_job.children[child_key]['download_status'] = 'failed'
-            return False
-
-        # extract zip
-        try:
-            logger.debug(f"Extracting zip file: {zip_file_path}")
-            extract_path = os.path.dirname(zip_file_path)
-            with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
-                zip_ref.extractall(extract_path)
-                logger.info(f"Successfully extracted zip to: {extract_path}")
-            os.remove(zip_file_path)
-            local_job.children[child_key]['download_status'] = 'complete'
-        except Exception as e:
-            logger.exception(f"Exception extracting zip file: {e}")
-            local_job.children[child_key]['download_status'] = 'failed'
-
-        return local_job.children[child_key].get('download_status', None) == 'complete'
+        download_success = download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
+        if subjob_data['status'] == 'completed' and download_success:
+            local_job.children[subjob_key]['download_status'] = 'completed'
 
     @classmethod
-    def wait_for_subjobs(cls, local_job):
-        # todo: rewrite this method
-        logger.debug(f"Waiting for subjobs for job {local_job}")
-        local_job.status = RenderStatus.WAITING_FOR_SUBJOBS
+    def wait_for_subjobs(cls, parent_job):
+        """Check the status of subjobs and waits until they are all finished. Download rendered frames from subjobs
+        when they are completed.
+
+        Args:
+            parent_job: Worker object that has child jobs
+
+        Returns:
+        """
+        logger.debug(f"Waiting for subjobs for job {parent_job}")
+        parent_job.status = RenderStatus.WAITING_FOR_SUBJOBS
         statuses_to_download = [RenderStatus.CANCELLED, RenderStatus.ERROR, RenderStatus.COMPLETED]
 
         def subjobs_not_downloaded():
-            return {k: v for k, v in local_job.children.items() if 'download_status' not in v or
+            return {k: v for k, v in parent_job.children.items() if 'download_status' not in v or
                     v['download_status'] == 'working' or v['download_status'] is None}
 
-        logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {local_job.id}')
+        logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {parent_job.id}')
 
-        while len(subjobs_not_downloaded()):
-            for child_key, subjob_cached_data in subjobs_not_downloaded().items():
+        server_delay = 10
+        sleep_counter = 0
+        while parent_job.status == RenderStatus.WAITING_FOR_SUBJOBS:
 
-                subjob_id = child_key.split('@')[0]
-                subjob_hostname = child_key.split('@')[-1]
+            if sleep_counter % server_delay == 0:  # only ping servers every x seconds
+                for child_key, subjob_cached_data in subjobs_not_downloaded().items():
 
-                # Fetch info from server and handle failing case
-                subjob_data = RenderServerProxy(subjob_hostname).get_job_info(subjob_id)
-                if not subjob_data:
-                    logger.warning(f"No response from: {subjob_hostname}")
-                    # todo: handle timeout / missing server situations
-                    continue
+                    subjob_id = child_key.split('@')[0]
+                    subjob_hostname = child_key.split('@')[-1]
 
-                # Update parent job cache but keep the download status
-                download_status = local_job.children[child_key].get('download_status', None)
-                local_job.children[child_key] = subjob_data
-                local_job.children[child_key]['download_status'] = download_status
+                    # Fetch info from server and handle failing case
+                    subjob_data = RenderServerProxy(subjob_hostname).get_job_info(subjob_id)
+                    if not subjob_data:
+                        logger.warning(f"No response from {subjob_hostname}")
+                        # timeout / missing server situations
+                        parent_job.children[child_key]['download_status'] = f'error: No response from {subjob_hostname}'
+                        continue
 
-                status = string_to_status(subjob_data.get('status', ''))
-                status_msg = f"Subjob {child_key} | {status} | " \
-                             f"{float(subjob_data.get('percent_complete')) * 100.0}%"
-                logger.debug(status_msg)
+                    # Update parent job cache but keep the download status
+                    download_status = parent_job.children[child_key].get('download_status', None)
+                    parent_job.children[child_key] = subjob_data
+                    parent_job.children[child_key]['download_status'] = download_status
 
-                # Still working in another thread - keep waiting
-                if download_status == 'working':
-                    continue
+                    status = string_to_status(subjob_data.get('status', ''))
+                    status_msg = f"Subjob {child_key} | {status} | " \
+                                 f"{float(subjob_data.get('percent_complete')) * 100.0}%"
+                    logger.debug(status_msg)
 
-                # Check if job is finished, but has not had files copied yet over yet
-                if download_status is None and subjob_data['file_count'] and status in statuses_to_download:
-                    try:
-                        cls.download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
-                    except Exception as e:
-                        logger.error(f"Error downloading missing frames from subjob: {e}")
+                    # Check if job is finished, but has not had files copied yet over yet
+                    if download_status is None and subjob_data['file_count'] and status in statuses_to_download:
+                        try:
+                            download_missing_frames_from_subjob(parent_job, subjob_id, subjob_hostname)
+                            parent_job.children[child_key]['download_status'] = 'complete'
+                        except Exception as e:
+                            logger.error(f"Error downloading missing frames from subjob: {e}")
+                            parent_job.children[child_key]['download_status'] = 'error: {}'
 
-                # Any finished jobs not successfully downloaded at this point are skipped
-                if local_job.children[child_key].get('download_status', None) is None and \
-                        status in statuses_to_download:
-                    logger.warning(f"Skipping waiting on downloading from subjob: {child_key}")
-                    local_job.children[child_key]['download_status'] = 'skipped'
+                    # Any finished jobs not successfully downloaded at this point are skipped
+                    if parent_job.children[child_key].get('download_status', None) is None and \
+                            status in statuses_to_download:
+                        logger.warning(f"Skipping waiting on downloading from subjob: {child_key}")
+                        parent_job.children[child_key]['download_status'] = 'skipped'
 
             if subjobs_not_downloaded():
                 logger.debug(f"Waiting on {len(subjobs_not_downloaded())} subjobs on "
                              f"{', '.join(list(subjobs_not_downloaded().keys()))}")
-                time.sleep(5)
+                time.sleep(1)
+                sleep_counter += 1
+            else:  # exit the loop
+                parent_job.status = RenderStatus.RUNNING
 
     # --------------------------------------------
     # Creating Subjobs
     # --------------------------------------------
 
     @classmethod
-    def split_into_subjobs_async(cls, parent_worker, job_data, project_path, system_os=None):
+    def split_into_subjobs_async(cls, parent_worker, new_job_attributes, project_path, system_os=None):
         # todo: I don't love this
         parent_worker.status = RenderStatus.CONFIGURING
-        cls.background_worker = threading.Thread(target=cls.split_into_subjobs, args=(parent_worker, job_data,
+        cls.background_worker = threading.Thread(target=cls.split_into_subjobs, args=(parent_worker, new_job_attributes,
                                                                                       project_path, system_os))
         cls.background_worker.start()
 
     @classmethod
-    def split_into_subjobs(cls, parent_worker, job_data, project_path, system_os=None, specific_servers=None):
+    def split_into_subjobs(cls, parent_worker, new_job_attributes, project_path, system_os=None, specific_servers=None):
         """
         Splits a job into subjobs and distributes them among available servers.
 
@@ -358,43 +302,43 @@ class DistributedJobManager:
         subjob.
 
         Args:
-            parent_worker (Worker): The worker that is handling the job.
-            job_data (dict): The data for the job to be split.
-            project_path (str): The path to the project associated with the job.
-            system_os (str, optional): The operating system of the servers. Default is any OS.
+            parent_worker (Worker): The parent job what we're creating the subjobs for.
+            new_job_attributes (dict): Dict of desired attributes for new job (frame count, renderer, output path, etc)
+            project_path (str): The path to the project.
+            system_os (str, optional): Required OS. Default is any.
             specific_servers (list, optional): List of specific servers to split work between. Defaults to all found.
         """
 
         # Check availability
-        parent_worker.status = RenderStatus.CONFIGURING
-        available_servers = specific_servers if specific_servers else cls.find_available_servers(parent_worker.renderer, system_os)
-        logger.debug(f"Splitting into subjobs - Available servers: {available_servers}")
-        all_subjob_server_data = cls.distribute_server_work(parent_worker.start_frame, parent_worker.end_frame, available_servers)
+        available_servers = specific_servers if specific_servers else cls.find_available_servers(parent_worker.renderer,
+                                                                                                 system_os)
+        # skip if theres no external servers found
+        external_servers = [x for x in available_servers if x['hostname'] != parent_worker.hostname]
+        if not external_servers:
+            parent_worker.status = RenderStatus.NOT_STARTED
+            return
+
+        logger.debug(f"Splitting into subjobs - Available servers: {[x['hostname'] for x in available_servers]}")
+        all_subjob_server_data = distribute_server_work(parent_worker.start_frame, parent_worker.end_frame, available_servers)
 
         # Prep and submit these sub-jobs
         logger.info(f"Job {parent_worker.id} split plan: {all_subjob_server_data}")
         try:
             for subjob_data in all_subjob_server_data:
                 subjob_hostname = subjob_data['hostname']
-                if subjob_hostname != parent_worker.hostname:
-                    post_results = cls.__create_subjob(job_data, project_path, subjob_data, subjob_hostname,
-                                                       parent_worker)
-                    if not post_results.ok:
-                        ValueError(f"Failed to create subjob on {subjob_hostname}")
+                post_results = cls.__create_subjob(new_job_attributes, project_path, subjob_data, subjob_hostname,
+                                                   parent_worker)
+                if not post_results.ok:
+                    ValueError(f"Failed to create subjob on {subjob_hostname}")
 
-                    # save child info
-                    submission_results = post_results.json()[0]
-                    child_key = f"{submission_results['id']}@{subjob_hostname}"
-                    parent_worker.children[child_key] = submission_results
-                else:
-                    # truncate parent render_job
-                    parent_worker.start_frame = max(subjob_data['frame_range'][0], parent_worker.start_frame)
-                    parent_worker.end_frame = min(subjob_data['frame_range'][-1], parent_worker.end_frame)
-                    logger.info(f"Local job now rendering from {parent_worker.start_frame} to {parent_worker.end_frame}")
+                # save child info
+                submission_results = post_results.json()[0]
+                child_key = f"{submission_results['id']}@{subjob_hostname}"
+                parent_worker.children[child_key] = submission_results
 
             # start subjobs
-            logger.debug(f"Created {len(all_subjob_server_data) - 1} subjobs successfully")
-            parent_worker.name = f"{parent_worker.name}[{parent_worker.start_frame}-{parent_worker.end_frame}]"
+            logger.debug(f"Created {len(all_subjob_server_data)} subjobs successfully")
+            parent_worker.name = f"{parent_worker.name} (Parent)"
             parent_worker.status = RenderStatus.NOT_STARTED  # todo: this won't work with scheduled starts
         except Exception as e:
             # cancel all the subjobs
@@ -403,8 +347,9 @@ class DistributedJobManager:
             RenderServerProxy(parent_worker.hostname).cancel_job(parent_worker.id, confirm=True)
 
     @staticmethod
-    def __create_subjob(job_data, project_path, server_data, server_hostname, parent_worker):
-        subjob = job_data.copy()
+    def __create_subjob(new_job_attributes, project_path, server_data, server_hostname, parent_worker):
+        """Convenience method to create subjobs for a parent worker"""
+        subjob = new_job_attributes.copy()
         subjob['name'] = f"{parent_worker.name}[{server_data['frame_range'][0]}-{server_data['frame_range'][-1]}]"
         subjob['parent'] = f"{parent_worker.id}@{parent_worker.hostname}"
         subjob['start_frame'] = server_data['frame_range'][0]
@@ -420,126 +365,6 @@ class DistributedJobManager:
     # Server Handling
     # --------------------------------------------
 
-    @staticmethod
-    def distribute_server_work(start_frame, end_frame, available_servers, method='cpu_benchmark'):
-        """
-        Splits the frame range among available servers proportionally based on their performance (CPU count).
-
-        Args:
-            start_frame (int): The start frame number of the animation to be rendered.
-            end_frame (int): The end frame number of the animation to be rendered.
-            available_servers (list): A list of available server dictionaries. Each server dictionary should include
-            'hostname' and 'cpu_count' keys (see find_available_servers).
-            method (str, optional): Specifies the distribution method. Possible values are 'cpu_benchmark', 'cpu_count'
-            and 'evenly'.
-            Defaults to 'cpu_benchmark'.
-
-        Returns:
-            list: A list of server dictionaries where each dictionary includes the frame range and total number of
-            frames to be rendered by the server.
-        """
-        # Calculate respective frames for each server
-        def divide_frames_by_cpu_count(frame_start, frame_end, servers):
-            total_frames = frame_end - frame_start + 1
-            total_cpus = sum(server['cpu_count'] for server in servers)
-
-            frame_ranges = {}
-            current_frame = frame_start
-            allocated_frames = 0
-
-            for i, server in enumerate(servers):
-                if i == len(servers) - 1:  # if it's the last server
-                    # Give all remaining frames to the last server
-                    num_frames = total_frames - allocated_frames
-                else:
-                    num_frames = round((server['cpu_count'] / total_cpus) * total_frames)
-                    allocated_frames += num_frames
-
-                frame_end_for_server = current_frame + num_frames - 1
-
-                if current_frame <= frame_end_for_server:
-                    frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
-                current_frame = frame_end_for_server + 1
-
-            return frame_ranges
-
-        def divide_frames_by_benchmark(frame_start, frame_end, servers):
-
-            def fetch_benchmark(server):
-                try:
-                    benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}'
-                                             f'/api/cpu_benchmark').text
-                    server['cpu_benchmark'] = benchmark
-                    logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}')
-                except requests.exceptions.RequestException as e:
-                    logger.error(f'Error fetching benchmark for {server["hostname"]}: {e}')
-
-            # Number of threads to use (can adjust based on your needs or number of servers)
-            threads = len(servers)
-
-            with ThreadPoolExecutor(max_workers=threads) as executor:
-                executor.map(fetch_benchmark, servers)
-
-            total_frames = frame_end - frame_start + 1
-            total_performance = sum(int(server['cpu_benchmark']) for server in servers)
-
-            frame_ranges = {}
-            current_frame = frame_start
-            allocated_frames = 0
-
-            for i, server in enumerate(servers):
-                if i == len(servers) - 1:  # if it's the last server
-                    # Give all remaining frames to the last server
-                    num_frames = total_frames - allocated_frames
-                else:
-                    num_frames = round((int(server['cpu_benchmark']) / total_performance) * total_frames)
-                    allocated_frames += num_frames
-
-                frame_end_for_server = current_frame + num_frames - 1
-
-                if current_frame <= frame_end_for_server:
-                    frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
-                current_frame = frame_end_for_server + 1
-
-            return frame_ranges
-
-        def divide_frames_equally(frame_start, frame_end, servers):
-            frame_range = frame_end - frame_start + 1
-            frames_per_server = frame_range // len(servers)
-            leftover_frames = frame_range % len(servers)
-
-            frame_ranges = {}
-            current_start = frame_start
-            for i, server in enumerate(servers):
-                current_end = current_start + frames_per_server - 1
-                if leftover_frames > 0:
-                    current_end += 1
-                    leftover_frames -= 1
-                if current_start <= current_end:
-                    frame_ranges[server['hostname']] = (current_start, current_end)
-                current_start = current_end + 1
-
-            return frame_ranges
-
-        if len(available_servers) == 1:
-            breakdown = {available_servers[0]['hostname']: (start_frame, end_frame)}
-        else:
-            logger.debug(f'Splitting between {len(available_servers)} servers by {method} method')
-            if method == 'evenly':
-                breakdown = divide_frames_equally(start_frame, end_frame, available_servers)
-            elif method == 'cpu_benchmark':
-                breakdown = divide_frames_by_benchmark(start_frame, end_frame, available_servers)
-            elif method == 'cpu_count':
-                breakdown = divide_frames_by_cpu_count(start_frame, end_frame, available_servers)
-            else:
-                raise ValueError(f"Invalid distribution method: {method}")
-
-        server_breakdown = [server for server in available_servers if breakdown.get(server['hostname']) is not None]
-        for server in server_breakdown:
-            server['frame_range'] = breakdown[server['hostname']]
-            server['total_frames'] = breakdown[server['hostname']][-1] - breakdown[server['hostname']][0] + 1
-        return server_breakdown
-
     @staticmethod
     def find_available_servers(engine_name, system_os=None):
         """
@@ -569,6 +394,6 @@ if __name__ == '__main__':
     time.sleep(2)
     available_servers = DistributedJobManager.find_available_servers('blender')
     print(f"AVAILABLE SERVERS ({len(available_servers)}): {available_servers}")
-    # results = DistributedJobManager.distribute_server_work(1, 100, available_servers)
+    # results = distribute_server_work(1, 100, available_servers)
     # print(f"RESULTS: {results}")
     ZeroconfServer.stop()

src/engines/blender/blender_engine.py

@@ -1,12 +1,13 @@
 import json
 import re
-from concurrent.futures import ThreadPoolExecutor
 
 from src.engines.core.base_engine import *
 from src.utilities.misc_helper import system_safe_path
 
 logger = logging.getLogger()
 
+_creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
+
 
 class Blender(BaseRenderEngine):
 
@@ -23,12 +24,12 @@ class Blender(BaseRenderEngine):
         from src.engines.blender.blender_worker import BlenderRenderWorker
         return BlenderRenderWorker
 
-    def ui_options(self):
-        from src.engines.blender.blender_ui import BlenderUI
-        return BlenderUI.get_options(self)
-
     @staticmethod
-    def supported_extensions():
+    def ui_options(system_info):
+        from src.engines.blender.blender_ui import BlenderUI
+        return BlenderUI.get_options(system_info)
+
+    def supported_extensions(self):
         return ['blend']
 
     def version(self):
@@ -36,7 +37,8 @@ class Blender(BaseRenderEngine):
         try:
             render_path = self.renderer_path()
             if render_path:
-                ver_out = subprocess.check_output([render_path, '-v'], timeout=SUBPROCESS_TIMEOUT)
+                ver_out = subprocess.check_output([render_path, '-v'], timeout=SUBPROCESS_TIMEOUT,
+                                                  creationflags=_creationflags)
                 version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip()
         except Exception as e:
             logger.error(f'Failed to get Blender version: {e}')
@@ -51,9 +53,11 @@ class Blender(BaseRenderEngine):
         if os.path.exists(project_path):
             try:
                 return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression],
-                                      capture_output=True, timeout=timeout)
+                                      capture_output=True, timeout=timeout, creationflags=_creationflags)
             except Exception as e:
-                logger.error(f"Error running python expression in blender: {e}")
+                err_msg = f"Error running python expression in blender: {e}"
+                logger.error(err_msg)
+                raise ChildProcessError(err_msg)
         else:
             raise FileNotFoundError(f'Project file not found: {project_path}')
 
@@ -68,9 +72,16 @@ class Blender(BaseRenderEngine):
             command = [self.renderer_path(), '-b', '--python', script_path]
             if project_path:
                 command.insert(2, project_path)
-            return subprocess.run(command, capture_output=True, timeout=timeout)
+            result = subprocess.run(command, capture_output=True, timeout=timeout, creationflags=_creationflags)
+            return result
+        except subprocess.TimeoutExpired:
+            err_msg = f"Timed out after {timeout}s while running python script in blender: {script_path}"
+            logger.error(err_msg)
+            raise TimeoutError(err_msg)
         except Exception as e:
-            logger.exception(f"Error running python script in blender: {e}")
+            err_msg = f"Error running python script in blender: {e}"
+            logger.error(err_msg)
+            raise ChildProcessError(err_msg)
 
     def get_project_info(self, project_path, timeout=10):
         scene_info = {}
@@ -87,10 +98,12 @@ class Blender(BaseRenderEngine):
                 elif line.startswith('Error'):
                     logger.error(f"get_scene_info error: {line.strip()}")
         except Exception as e:
-            logger.error(f'Error getting file details for .blend file: {e}')
+            msg = f'Error getting file details for .blend file: {e}'
+            logger.error(msg)
+            raise ChildProcessError(msg)
         return scene_info
 
-    def pack_project_file(self, project_path, timeout=30):
+    def pack_project_file(self, project_path, timeout=None):
         # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935
         try:
             logger.info(f"Starting to pack Blender file: {project_path}")
@@ -113,11 +126,13 @@ class Blender(BaseRenderEngine):
                 logger.info(f'Blender file packed successfully to {new_path}')
                 return new_path
         except Exception as e:
-            logger.error(f'Error packing .blend file: {e}')
+            msg = f'Error packing .blend file: {e}'
+            logger.error(msg)
+            raise ChildProcessError(msg)
         return None
 
     def get_arguments(self):
-        help_text = subprocess.check_output([self.renderer_path(), '-h']).decode('utf-8')
+        help_text = subprocess.check_output([self.renderer_path(), '-h'], creationflags=_creationflags).decode('utf-8')
         lines = help_text.splitlines()
 
         options = {}
@@ -149,13 +164,7 @@ class Blender(BaseRenderEngine):
         return options
 
     def system_info(self):
-        with ThreadPoolExecutor() as executor:
-            future_render_devices = executor.submit(self.get_render_devices)
-            future_engines = executor.submit(self.supported_render_engines)
-            render_devices = future_render_devices.result()
-            engines = future_engines.result()
-
-        return {'render_devices': render_devices, 'engines': engines}
+        return {'render_devices': self.get_render_devices()}
 
     def get_render_devices(self):
         script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'scripts', 'get_system_info.py')
@@ -171,12 +180,12 @@ class Blender(BaseRenderEngine):
 
     def supported_render_engines(self):
         engine_output = subprocess.run([self.renderer_path(), '-E', 'help'], timeout=SUBPROCESS_TIMEOUT,
-                                       capture_output=True).stdout.decode('utf-8').strip()
+                                       capture_output=True, creationflags=_creationflags).stdout.decode('utf-8').strip()
         render_engines = [x.strip() for x in engine_output.split('Blender Engine Listing:')[-1].strip().splitlines()]
         return render_engines
 
     def perform_presubmission_tasks(self, project_path):
-        packed_path = self.pack_project_file(project_path, timeout=30)
+        packed_path = self.pack_project_file(project_path, timeout=120)
         return packed_path
 
 

src/engines/blender/blender_ui.py

@@ -1,9 +1,9 @@
 
 class BlenderUI:
     @staticmethod
-    def get_options(instance):
+    def get_options(system_info):
         options = [
-            {'name': 'engine', 'options': instance.supported_render_engines()},
+            {'name': 'engine', 'options': system_info.get('engines', [])},
             {'name': 'render_device', 'options': ['Any', 'GPU', 'CPU']},
         ]
         return options

src/engines/blender/blender_worker.py

@@ -40,13 +40,11 @@ class BlenderRenderWorker(BaseRenderWorker):
         if custom_camera:
             python_exp = python_exp + f"bpy.context.scene.camera = bpy.data.objects['{custom_camera}'];"
 
-        # Setup Render Engines
-        self.args['engine'] = self.args.get('engine', 'CYCLES').upper()  # set default render engine
-        # Configure Cycles
-        if self.args['engine'] == 'CYCLES':
-            # Set Render Device (gpu/cpu/any)
+        # Set Render Device (gpu/cpu/any)
+        blender_engine = self.args.get('engine', 'BLENDER_EEVEE').upper()
+        if blender_engine == 'CYCLES':
             render_device = self.args.get('render_device', 'any').lower()
-            if render_device not in ['any', 'gpu', 'cpu']:
+            if render_device not in {'any', 'gpu', 'cpu'}:
                 raise AttributeError(f"Invalid Cycles render device: {render_device}")
 
             use_gpu = render_device in {'any', 'gpu'}

src/engines/core/base_downloader.py

@@ -1,9 +1,7 @@
 import logging
 import os
 import shutil
-import tarfile
 import tempfile
-import zipfile
 
 import requests
 from tqdm import tqdm
@@ -12,26 +10,150 @@ logger = logging.getLogger()
 
 
 class EngineDownloader:
+    """A class responsible for downloading and extracting rendering engines from publicly available URLs.
+
+    Attributes:
+        supported_formats (list[str]): A list of file formats supported by the downloader.
+    """
 
     supported_formats = ['.zip', '.tar.xz', '.dmg']
 
     def __init__(self):
         pass
 
+    # --------------------------------------------
+    # Required Overrides for Subclasses:
+    # --------------------------------------------
+
     @classmethod
     def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
-        raise NotImplementedError  # implement this method in your engine subclass
+        """
+        Finds the most recent version of the rendering engine available for download.
+
+        This method should be overridden in a subclass to implement the logic for determining
+        the most recent version of the rendering engine, optionally filtering by long-term
+        support (LTS) versions, the operating system, and CPU architecture.
+
+        Args:
+            system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
+            cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
+            lts_only (bool, optional): Limit the search to LTS (long-term support) versions only. Default is False.
+
+        Returns:
+            dict: A dict with the following keys:
+                - 'cpu' (str): The CPU architecture.
+                - 'system_os' (str): The operating system.
+                - 'file' (str): The filename of the version's download file.
+                - 'url' (str): The remote URL for downloading the version.
+                - 'version' (str): The version number.
+
+        Raises:
+            NotImplementedError: If the method is not overridden in a subclass.
+        """
+        raise NotImplementedError(f"find_most_recent_version not implemented for {cls.__class__.__name__}")
 
     @classmethod
     def version_is_available_to_download(cls, version, system_os=None, cpu=None):
-        raise NotImplementedError  # implement this method in your engine subclass
+        """Checks if a requested version of the rendering engine is available for download.
+
+        This method should be overridden in a subclass to implement the logic for determining
+        whether a given version of the rendering engine is available for download, based on the
+        operating system and CPU architecture.
+
+        Args:
+            version (str): The requested renderer version to download.
+            system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
+            cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
+
+        Returns:
+            bool: True if the version is available for download, False otherwise.
+
+        Raises:
+            NotImplementedError: If the method is not overridden in a subclass.
+        """
+        raise NotImplementedError(f"version_is_available_to_download not implemented for {cls.__class__.__name__}")
 
     @classmethod
     def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
-        raise NotImplementedError  # implement this method in your engine subclass
+        """Downloads the requested version of the rendering engine to the given download location.
+
+        This method should be overridden in a subclass to implement the logic for downloading
+        a specific version of the rendering engine. The method is intended to handle the
+        downloading process based on the version, operating system, CPU architecture, and
+        timeout parameters.
+
+        Args:
+            version (str): The requested renderer version to download.
+            download_location (str): The directory where the engine should be downloaded.
+            system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
+            cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
+            timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
+
+        Raises:
+            NotImplementedError: If the method is not overridden in a subclass.
+        """
+        raise NotImplementedError(f"download_engine not implemented for {cls.__class__.__name__}")
+
+    # --------------------------------------------
+    # Optional Overrides for Subclasses:
+    # --------------------------------------------
+
+    @classmethod
+    def all_versions(cls, system_os=None, cpu=None):
+        """Retrieves a list of available versions of the software for a specific operating system and CPU architecture.
+
+        This method fetches all available versions for the given operating system and CPU type, constructing
+        a list of dictionaries containing details such as the version, CPU architecture, system OS, and the
+        remote URL for downloading each version.
+
+        Args:
+            system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
+            cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
+
+        Returns:
+            list[dict]: A list of dictionaries, each containing:
+                - 'cpu' (str): The CPU architecture.
+                - 'file' (str): The filename of the version's download file.
+                - 'system_os' (str): The operating system.
+                - 'url' (str): The remote URL for downloading the version.
+                - 'version' (str): The version number.
+        """
+        return []
+
+    # --------------------------------------------
+    # Do Not Override These Methods:
+    # --------------------------------------------
 
     @classmethod
     def download_and_extract_app(cls, remote_url, download_location, timeout=120):
+        """Downloads an application from the given remote URL and extracts it to the specified location.
+
+        This method handles the downloading of the application, supports multiple archive formats,
+        and extracts the contents to the specified `download_location`. It also manages temporary
+        files and logs progress throughout the process.
+
+        Args:
+            remote_url (str): The URL of the application to download.
+            download_location (str): The directory where the application should be extracted.
+            timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
+
+        Returns:
+            str: The path to the directory where the application was extracted.
+
+        Raises:
+            Exception: Catches and logs any exceptions that occur during the download or extraction process.
+
+        Supported Formats:
+            - `.tar.xz`: Extracted using the `tarfile` module.
+            - `.zip`: Extracted using the `zipfile` module.
+            - `.dmg`: macOS disk image files, handled using the `dmglib` library.
+            - Other formats will result in an error being logged.
+
+        Notes:
+            - If the application already exists in the `download_location`, the method will log an error
+              and return without downloading or extracting.
+            - Temporary files created during the download process are cleaned up after completion.
+        """
 
         # Create a temp download directory
         temp_download_dir = tempfile.mkdtemp()
@@ -80,6 +202,7 @@ class EngineDownloader:
             # Extract the downloaded file
             # Process .tar.xz files
             if temp_downloaded_file_path.lower().endswith('.tar.xz'):
+                import tarfile
                 try:
                     with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
                         tar.extractall(path=download_location)
@@ -93,6 +216,7 @@ class EngineDownloader:
 
             # Process .zip files
             elif temp_downloaded_file_path.lower().endswith('.zip'):
+                import zipfile
                 try:
                     with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
                         zip_ref.extractall(download_location)

src/engines/core/base_engine.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import platform
 import subprocess
 
 logger = logging.getLogger()
@@ -7,19 +8,139 @@ SUBPROCESS_TIMEOUT = 5
 
 
 class BaseRenderEngine(object):
+    """Base class for render engines. This class provides common functionality and structure for various rendering
+    engines. Create subclasses and override the methods marked below to add additional renderers
+
+    Attributes:
+        install_paths (list): A list of default installation paths where the render engine
+        might be found. This list can be populated with common paths to help locate the
+        executable on different operating systems or environments.
+    """
 
     install_paths = []
-    supported_extensions = []
+
+    # --------------------------------------------
+    # Required Overrides for Subclasses:
+    # --------------------------------------------
 
     def __init__(self, custom_path=None):
         self.custom_renderer_path = custom_path
         if not self.renderer_path() or not os.path.exists(self.renderer_path()):
-            raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance")
+            raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance: {self.renderer_path()}")
 
         if not os.access(self.renderer_path(), os.X_OK):
             logger.warning(f"Path is not executable. Setting permissions to 755 for {self.renderer_path()}")
             os.chmod(self.renderer_path(), 0o755)
 
+    def version(self):
+        """Return the version number as a string.
+
+        Returns:
+            str: Version number.
+
+        Raises:
+            NotImplementedError: If not overridden.
+        """
+        raise NotImplementedError(f"version not implemented for {self.__class__.__name__}")
+
+    def get_project_info(self, project_path, timeout=10):
+        """Extracts detailed project information from the given project path.
+
+        Args:
+            project_path (str): The path to the project file.
+            timeout (int, optional): The maximum time (in seconds) to wait for the operation. Default is 10 seconds.
+
+        Returns:
+            dict: A dictionary containing project information (subclasses should define the structure).
+
+        Raises:
+            NotImplementedError: If the method is not overridden in a subclass.
+        """
+        raise NotImplementedError(f"get_project_info not implemented for {self.__class__.__name__}")
+
+    @classmethod
+    def get_output_formats(cls):
+        """Returns a list of available output formats supported by the renderer.
+
+        Returns:
+            list[str]: A list of strings representing the available output formats.
+        """
+        raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
+
+    @staticmethod
+    def worker_class():  # override when subclassing to link worker class
+        raise NotImplementedError("Worker class not implemented")
+
+    # --------------------------------------------
+    # Optional Overrides for Subclasses:
+    # --------------------------------------------
+
+    def supported_extensions(self):
+        """
+        Returns:
+            list[str]: list of supported extensions
+        """
+        return []
+
+    def get_help(self):
+        """Retrieves the help documentation for the renderer.
+
+        This method runs the renderer's help command (default: '-h') and captures the output.
+        Override this method if the renderer uses a different help flag.
+
+        Returns:
+            str: The help documentation as a string.
+
+        Raises:
+            FileNotFoundError: If the renderer path is not found.
+        """
+        path = self.renderer_path()
+        if not path:
+            raise FileNotFoundError("renderer path not found")
+        creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
+        help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
+                                           timeout=SUBPROCESS_TIMEOUT, creationflags=creationflags).decode('utf-8')
+        return help_doc
+
+    def system_info(self):
+        """Return additional information about the system specfic to the engine (configured GPUs, render engines, etc)
+
+        Returns:
+            dict: A dictionary with engine-specific system information
+        """
+        return {}
+
+    def perform_presubmission_tasks(self, project_path):
+        """Perform any pre-submission tasks on a project file before uploading it to a server (pack textures, etc.)
+
+        Override this method to:
+            1. Copy the project file to a temporary location (DO NOT MODIFY ORIGINAL PATH).
+            2. Perform additional modifications or tasks.
+            3. Return the path to the modified project file.
+
+        Args:
+            project_path (str): The original project file path.
+
+        Returns:
+            str: The path to the modified project file.
+        """
+        return project_path
+
+    def get_arguments(self):
+        pass
+
+    @staticmethod
+    def downloader():  # override when subclassing if using a downloader class
+        return None
+
+    @staticmethod
+    def ui_options(system_info):  # override to return options for ui
+        return {}
+
+    # --------------------------------------------
+    # Do Not Override These Methods:
+    # --------------------------------------------
+
     def renderer_path(self):
         return self.custom_renderer_path or self.default_renderer_path()
 
@@ -39,41 +160,3 @@ class BaseRenderEngine(object):
         except Exception as e:
             logger.exception(e)
         return path
-
-    def version(self):
-        raise NotImplementedError("version not implemented")
-
-    @staticmethod
-    def downloader():  # override when subclassing if using a downloader class
-        return None
-
-    @staticmethod
-    def worker_class():  # override when subclassing to link worker class
-        raise NotImplementedError("Worker class not implemented")
-
-    def ui_options(self):  # override to return options for ui
-        return {}
-
-    def get_help(self):  # override if renderer uses different help flag
-        path = self.renderer_path()
-        if not path:
-            raise FileNotFoundError("renderer path not found")
-        help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
-                                           timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
-        return help_doc
-
-    def get_project_info(self, project_path, timeout=10):
-        raise NotImplementedError(f"get_project_info not implemented for {self.__name__}")
-
-    @classmethod
-    def get_output_formats(cls):
-        raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
-
-    def get_arguments(self):
-        pass
-
-    def system_info(self):
-        pass
-
-    def perform_presubmission_tasks(self, project_path):
-        return project_path

src/engines/core/base_worker.py

@@ -3,6 +3,7 @@ import io
 import json
 import logging
 import os
+import signal
 import subprocess
 import threading
 import time
@@ -47,6 +48,10 @@ class BaseRenderWorker(Base):
 
     engine = None
 
+    # --------------------------------------------
+    # Required Overrides for Subclasses:
+    # --------------------------------------------
+
     def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None,
                  name=None):
 
@@ -56,7 +61,7 @@ class BaseRenderWorker(Base):
                 logger.error(err_meg)
                 raise ValueError(err_meg)
         if not self.engine:
-            raise NotImplementedError("Engine not defined")
+            raise NotImplementedError(f"Engine not defined for {self.__class__.__name__}")
 
         def generate_id():
             import uuid
@@ -90,7 +95,7 @@ class BaseRenderWorker(Base):
         self.end_time = None
 
         # History
-        self.status = RenderStatus.NOT_STARTED
+        self.status = RenderStatus.CONFIGURING
         self.warnings = []
         self.errors = []
 
@@ -102,6 +107,50 @@ class BaseRenderWorker(Base):
         self.__last_output_time = None
         self.watchdog_timeout = 120
 
+    def generate_worker_subprocess(self):
+        """Generate a return a list of the command line arguments necessary to perform requested job
+        
+        Returns:
+            list[str]: list of command line arguments
+        """
+        raise NotImplementedError("generate_worker_subprocess not implemented")
+
+    def _parse_stdout(self, line):
+        """Parses a line of standard output from the renderer.
+
+        This method should be overridden in a subclass to implement the logic for processing
+        and interpreting a single line of output from the renderer's standard output stream.
+
+        On frame completion, the subclass should:
+            1. Update value of self.current_frame
+            2. Call self._send_frame_complete_notification()
+
+        Args:
+            line (str): A line of text from the renderer's standard output.
+
+        Raises:
+            NotImplementedError: If the method is not overridden in a subclass.
+        """
+        raise NotImplementedError(f"_parse_stdout not implemented for {self.__class__.__name__}")
+
+    # --------------------------------------------
+    # Optional Overrides for Subclasses:
+    # --------------------------------------------
+
+    def percent_complete(self):
+        # todo: fix this
+        if self.status == RenderStatus.COMPLETED:
+            return 1.0
+        return 0
+
+    def post_processing(self):
+        """Override to perform any engine-specific postprocessing"""
+        pass
+
+    # --------------------------------------------
+    # Do Not Override These Methods:
+    # --------------------------------------------
+
     def __repr__(self):
         return f"<Job id:{self.id} p{self.priority} {self.renderer}-{self.renderer_version} '{self.name}' status:{self.status.value}>"
 
@@ -141,16 +190,13 @@ class BaseRenderWorker(Base):
         return generated_args
 
     def get_raw_args(self):
-        raw_args_string = self.args.get('raw', None)
+        raw_args_string = self.args.get('raw', '')
         raw_args = None
         if raw_args_string:
             import shlex
             raw_args = shlex.split(raw_args_string)
         return raw_args
 
-    def generate_worker_subprocess(self):
-        raise NotImplementedError("generate_worker_subprocess not implemented")
-
     def log_path(self):
         filename = (self.name or os.path.basename(self.input_path)) + '_' + \
                    self.date_created.strftime("%Y.%m.%d_%H.%M.%S") + '.log'
@@ -158,7 +204,7 @@ class BaseRenderWorker(Base):
 
     def start(self):
 
-        if self.status not in [RenderStatus.SCHEDULED, RenderStatus.NOT_STARTED]:
+        if self.status not in [RenderStatus.SCHEDULED, RenderStatus.NOT_STARTED, RenderStatus.CONFIGURING]:
             logger.error(f"Trying to start job with status: {self.status}")
             return
 
@@ -176,90 +222,129 @@ class BaseRenderWorker(Base):
             self.errors.append(msg)
             return
 
-        self.status = RenderStatus.RUNNING
+        self.status = RenderStatus.RUNNING if not self.children else RenderStatus.WAITING_FOR_SUBJOBS
         self.start_time = datetime.now()
         self.__thread.start()
 
+    # handle multiple attempts at running subprocess
+    def __run__subprocess_cycle(self, log_file):
+        subprocess_cmds = self.generate_subprocess()
+        initial_file_count = len(self.file_list())
+        failed_attempts = 0
+
+        log_file.write(f"Running command: {subprocess_cmds}\n")
+        log_file.write('=' * 80 + '\n\n')
+
+        while True:
+            # Log attempt #
+            if failed_attempts:
+                if failed_attempts >= self.maximum_attempts:
+                    err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
+                    logger.error(err_msg)
+                    self.status = RenderStatus.ERROR
+                    self.errors.append(err_msg)
+                    return
+                else:
+                    log_file.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
+                    logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
+                    self.status = RenderStatus.RUNNING
+
+            return_code = self.__setup_and_run_process(log_file, subprocess_cmds)
+
+            message = f"{'=' * 50}\n\n{self.engine.name()} render ended with code {return_code} " \
+                      f"after {self.time_elapsed()}\n\n"
+            log_file.write(message)
+
+            # don't try again if we've been cancelled
+            if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
+                return
+
+            # if file output hasn't increased, return as error, otherwise restart process.
+            file_count_has_increased = len(self.file_list()) > initial_file_count
+            if (self.status == RenderStatus.RUNNING) and file_count_has_increased and not return_code:
+                break
+
+            if return_code:
+                err_msg = f"{self.engine.name()} render failed with code {return_code}"
+                logger.error(err_msg)
+                self.errors.append(err_msg)
+
+            # handle instances where renderer exits ok but doesnt generate files
+            if not return_code and not file_count_has_increased:
+                err_msg = (f"{self.engine.name()} render exited ok, but file count has not increased. "
+                           f"Count is still {len(self.file_list())}")
+                log_file.write(f'Error: {err_msg}\n\n')
+                self.errors.append(err_msg)
+
+            # only count the attempt as failed if renderer creates no output - reset counter on successful output
+            failed_attempts = 0 if file_count_has_increased else failed_attempts + 1
+
+    def __run__wait_for_subjobs(self, logfile):
+        from src.distributed_job_manager import DistributedJobManager
+        DistributedJobManager.wait_for_subjobs(parent_job=self)
+
+    @staticmethod
+    def log_and_print(message, log_file, level='info'):
+        if level == 'debug':
+            logger.debug(message)
+        elif level == 'error':
+            logger.error(message)
+        else:
+            logger.info(message)
+        log_file.write(f"{message}\n")
+
     def __run(self):
-        logger.info(f'Starting {self.engine.name()} {self.renderer_version} Render for {self.input_path} | '
-                    f'Frame Count: {self.total_frames}')
 
         # Setup logging
         log_dir = os.path.dirname(self.log_path())
         os.makedirs(log_dir, exist_ok=True)
 
-        subprocess_cmds = self.generate_subprocess()
-        initial_file_count = len(self.file_list())
-        failed_attempts = 0
+        with open(self.log_path(), "a") as log_file:
 
-        with open(self.log_path(), "a") as f:
+            self.log_and_print(f"{self.start_time.isoformat()} - Starting "
+                               f"{self.engine.name()} {self.renderer_version} render job for {self.name} "
+                               f"({self.input_path})", log_file)
+            log_file.write(f"\n")
+            if not self.children:
+                self.__run__subprocess_cycle(log_file)
+            else:
+                self.__run__wait_for_subjobs(log_file)
 
-            f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.renderer_version} "
-                    f"render for {self.input_path}\n\n")
-            f.write(f"Running command: {subprocess_cmds}\n")
-            f.write('=' * 80 + '\n\n')
+            # Validate Output - End if missing frames
+            if self.status == RenderStatus.RUNNING:
+                file_list_length = len(self.file_list())
+                expected_list_length = (self.end_frame - self.start_frame + 1) if self.end_frame else 1
 
-            while True:
-                # Log attempt #
-                if failed_attempts:
-                    if failed_attempts >= self.maximum_attempts:
-                        err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
-                        logger.error(err_msg)
-                        self.status = RenderStatus.ERROR
-                        self.errors.append(err_msg)
-                        return
-                    else:
-                        f.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
-                        logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
-                        self.status = RenderStatus.RUNNING
+                msg = f"Frames: Expected ({expected_list_length}) vs actual ({file_list_length}) for {self}"
+                self.log_and_print(msg, log_file, 'debug')
 
-                return_code = self.__setup_and_run_process(f, subprocess_cmds)
+                if file_list_length not in (expected_list_length, 1):
+                    msg = f"Missing frames: Expected ({expected_list_length}) vs actual ({file_list_length})"
+                    self.log_and_print(msg, log_file, 'error')
+                    self.errors.append(msg)
+                    self.status = RenderStatus.ERROR
+                    # todo: create new subjob to generate missing frames
+
+            # cleanup and close if cancelled / error
+            if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
                 self.end_time = datetime.now()
+                message = f"{self.engine.name()} render ended with status '{self.status.value}' " \
+                          f"after {self.time_elapsed()}"
+                self.log_and_print(message, log_file)
+                log_file.close()
+                return
 
-                message = f"{'=' * 50}\n\n{self.engine.name()} render ended with code {return_code} " \
-                          f"after {self.time_elapsed()}\n\n"
-                f.write(message)
+            # Post Render Work
+            if not self.parent:
+                logger.debug(f"Starting post-processing work for {self}")
+                self.log_and_print(f"Starting post-processing work for {self}", log_file, 'debug')
+                self.post_processing()
+                self.log_and_print(f"Completed post-processing work for {self}", log_file, 'debug')
 
-                # Teardown
-                if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
-                    message = f"{self.engine.name()} render ended with status '{self.status}' " \
-                              f"after {self.time_elapsed()}"
-                    f.write(message)
-                    return
-
-                # if file output hasn't increased, return as error, otherwise restart process.
-                file_count_has_increased = len(self.file_list()) > initial_file_count
-                if (self.status == RenderStatus.RUNNING) and file_count_has_increased and not return_code:
-                    message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
-                               f"{self.time_elapsed()}\n")
-                    f.write(message)
-                    break
-
-                if return_code:
-                    err_msg = f"{self.engine.name()} render failed with code {return_code}"
-                    logger.error(err_msg)
-                    self.errors.append(err_msg)
-
-                # handle instances where renderer exits ok but doesnt generate files
-                if not return_code and not file_count_has_increased:
-                    err_msg = (f"{self.engine.name()} render exited ok, but file count has not increased. "
-                               f"Count is still {len(self.file_list())}")
-                    f.write(f'Error: {err_msg}\n\n')
-                    self.errors.append(err_msg)
-
-                # only count the attempt as failed if renderer creates no output - ignore error codes for now
-                if not file_count_has_increased:
-                    failed_attempts += 1
-
-        if self.children:
-            from src.distributed_job_manager import DistributedJobManager
-            DistributedJobManager.wait_for_subjobs(local_job=self)
-
-        # Post Render Work
-        logger.debug("Starting post-processing work")
-        self.post_processing()
-        self.status = RenderStatus.COMPLETED
-        logger.info(f"Render {self.id}-{self.name} completed successfully after {self.time_elapsed()}")
+            self.status = RenderStatus.COMPLETED
+            self.end_time = datetime.now()
+            message = f"Render {self.name} completed successfully after {self.time_elapsed()}"
+            self.log_and_print(message, log_file)
 
     def __setup_and_run_process(self, f, subprocess_cmds):
 
@@ -269,7 +354,7 @@ class BaseRenderWorker(Base):
                 time_since_last_update = time.time() - self.__last_output_time
                 if time_since_last_update > self.watchdog_timeout:
                     logger.error(f"Process for {self} terminated due to exceeding timeout ({self.watchdog_timeout}s)")
-                    self.__process.kill()
+                    self.__kill_process()
                     break
                 # logger.debug(f'Watchdog for {self} - Time since last update: {time_since_last_update}')
                 time.sleep(1)
@@ -282,8 +367,14 @@ class BaseRenderWorker(Base):
 
         try:
             # Start process and get updates
-            self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                                              universal_newlines=False)
+            if os.name == 'posix':  # linux / mac
+                self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                                  universal_newlines=False, preexec_fn=os.setsid)
+            else:  # windows
+                creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
+                self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                                  universal_newlines=False,
+                                                  creationflags=creationflags)
 
             # Start watchdog
             self.__last_output_time = time.time()
@@ -312,7 +403,7 @@ class BaseRenderWorker(Base):
             message = f'Uncaught error running render process: {e}'
             f.write(message)
             logger.exception(message)
-            self.__process.kill()
+            self.__kill_process()
 
         # let watchdog end before continuing - prevents multiple watchdogs running when process restarts
         if watchdog_thread.is_alive():
@@ -320,11 +411,29 @@ class BaseRenderWorker(Base):
 
         return return_code
 
-    def post_processing(self):
-        pass
+    def __kill_process(self):
+        try:
+            if self.__process.poll():
+                return
+            logger.debug(f"Trying to kill process {self.__process}")
+            self.__process.terminate()
+            self.__process.kill()
+            if os.name == 'posix':  # linux / macos
+                os.killpg(os.getpgid(self.__process.pid), signal.SIGTERM)
+                os.killpg(os.getpgid(self.__process.pid), signal.SIGKILL)
+            else:  # windows
+                parent = psutil.Process(self.__process.pid)
+                for child in parent.children(recursive=True):
+                    child.kill()
+            self.__process.wait(timeout=5)
+            logger.debug(f"Process ended with status {self.__process.poll()}")
+        except (ProcessLookupError, AttributeError, psutil.NoSuchProcess):
+            pass
+        except Exception as e:
+            logger.error(f"Error stopping the process: {e}")
 
     def is_running(self):
-        if self.__thread:
+        if hasattr(self, '__thread'):
             return self.__thread.is_alive()
         return False
 
@@ -335,15 +444,11 @@ class BaseRenderWorker(Base):
             self.stop(is_error=True)
 
     def stop(self, is_error=False):
-        if hasattr(self, '__process'):
-            try:
-                process = psutil.Process(self.__process.pid)
-                for proc in process.children(recursive=True):
-                    proc.kill()
-                process.kill()
-            except Exception as e:
-                logger.debug(f"Error stopping the process: {e}")
-        if self.status in [RenderStatus.RUNNING, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED]:
+        logger.debug(f"Stopping {self}")
+
+        # cleanup status
+        if self.status in [RenderStatus.RUNNING, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
+                           RenderStatus.CONFIGURING]:
             if is_error:
                 err_message = self.errors[-1] if self.errors else 'Unknown error'
                 logger.error(f"Halting render due to error: {err_message}")
@@ -351,13 +456,9 @@ class BaseRenderWorker(Base):
             else:
                 self.status = RenderStatus.CANCELLED
 
-    def percent_complete(self):
-        if self.status == RenderStatus.COMPLETED:
-            return 1.0
-        return 0
-
-    def _parse_stdout(self, line):
-        raise NotImplementedError("_parse_stdout not implemented")
+        self.__kill_process()
+        if self.is_running():  # allow the log files to close
+            self.__thread.join(timeout=5)
 
     def time_elapsed(self):
         return get_time_elapsed(self.start_time, self.end_time)
@@ -365,7 +466,11 @@ class BaseRenderWorker(Base):
     def file_list(self):
         try:
             job_dir = os.path.dirname(self.output_path)
-            file_list = [os.path.join(job_dir, file) for file in os.listdir(job_dir)]
+            file_list = [
+                os.path.join(job_dir, file)
+                for file in os.listdir(job_dir)
+                if not file.startswith('.')  # Ignore hidden files
+            ]
             file_list.sort()
             return file_list
         except FileNotFoundError:

src/engines/engine_manager.py

@@ -12,6 +12,9 @@ logger = logging.getLogger()
 
 
 class EngineManager:
+    """Class that manages different versions of installed renderers and handles fetching and downloading new versions,
+    if possible.
+    """
 
     engines_path = None
     download_tasks = []
@@ -27,7 +30,7 @@ class EngineManager:
                 return obj
 
     @classmethod
-    def get_engines(cls, filter_name=None):
+    def get_engines(cls, filter_name=None, include_corrupt=False):
 
         if not cls.engines_path:
             raise FileNotFoundError("Engine path is not set")
@@ -49,10 +52,8 @@ class EngineManager:
                 # Initialize binary_name with engine name
                 binary_name = result_dict['engine'].lower()
                 # Determine the correct binary name based on the engine and system_os
-                for eng in cls.supported_engines():
-                    if eng.name().lower() == result_dict['engine']:
-                        binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
-                        break
+                eng = cls.engine_with_name(result_dict['engine'])
+                binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
 
                 # Find the path to the binary file
                 path = next(
@@ -60,8 +61,16 @@ class EngineManager:
                      os.walk(system_safe_path(os.path.join(cls.engines_path, directory))) if binary_name in files),
                     None
                 )
-
                 result_dict['path'] = path
+
+                # fetch version number from binary - helps detect corrupted downloads
+                binary_version = eng(path).version()
+                if not binary_version:
+                    logger.warning(f"Possible corrupt {eng.name()} {result_dict['version']} install detected: {path}")
+                    if not include_corrupt:
+                        continue
+                result_dict['version'] = binary_version or 'error'
+
                 # Add the result dictionary to results if it matches the filter_name or if no filter is applied
                 if not filter_name or filter_name == result_dict['engine']:
                     results.append(result_dict)
@@ -69,10 +78,14 @@ class EngineManager:
             logger.warning(f"Cannot find local engines download directory: {e}")
 
         # add system installs to this list - use bg thread because it can be slow
-        def fetch_engine_details(eng):
+        def fetch_engine_details(eng, include_corrupt=False):
+            version = eng().version()
+            if not version and not include_corrupt:
+                return
+
             return {
                 'engine': eng.name(),
-                'version': eng().version(),
+                'version': version or 'error',
                 'system_os': current_system_os(),
                 'cpu': current_system_cpu(),
                 'path': eng.default_renderer_path(),
@@ -81,7 +94,7 @@ class EngineManager:
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
             futures = {
-                executor.submit(fetch_engine_details, eng): eng.name()
+                executor.submit(fetch_engine_details, eng, include_corrupt): eng.name()
                 for eng in cls.supported_engines()
                 if eng.default_renderer_path() and (not filter_name or filter_name == eng.name())
             }
@@ -94,8 +107,8 @@ class EngineManager:
         return results
 
     @classmethod
-    def all_versions_for_engine(cls, engine_name):
-        versions = cls.get_engines(filter_name=engine_name)
+    def all_versions_for_engine(cls, engine_name, include_corrupt=False):
+        versions = cls.get_engines(filter_name=engine_name, include_corrupt=include_corrupt)
         sorted_versions = sorted(versions, key=lambda x: x['version'], reverse=True)
         return sorted_versions
 
@@ -266,13 +279,24 @@ class EngineManager:
         _, extension = os.path.splitext(path)
         extension = extension.lower().strip('.')
         for engine in cls.supported_engines():
-            if extension in engine.supported_extensions():
+            if extension in engine().supported_extensions():
                 return engine
-        undefined_renderer_support = [x for x in cls.supported_engines() if not x.supported_extensions()]
+        undefined_renderer_support = [x for x in cls.supported_engines() if not x().supported_extensions()]
         return undefined_renderer_support[0]
 
 
 class EngineDownloadWorker(threading.Thread):
+    """A thread worker for downloading a specific version of a rendering engine.
+
+    This class handles the process of downloading a rendering engine in a separate thread,
+    ensuring that the download process does not block the main application.
+
+    Attributes:
+        engine (str): The name of the rendering engine to download.
+        version (str): The version of the rendering engine to download.
+        system_os (str, optional): The operating system for which to download the engine. Defaults to current OS type.
+        cpu (str, optional): Requested CPU architecture. Defaults to system CPU type.
+    """
     def __init__(self, engine, version, system_os=None, cpu=None):
         super().__init__()
         self.engine = engine

src/engines/ffmpeg/ffmpeg_engine.py

@@ -3,6 +3,8 @@ import re
 
 from src.engines.core.base_engine import *
 
+_creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
+
 
 class FFMPEG(BaseRenderEngine):
     binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
@@ -21,10 +23,9 @@ class FFMPEG(BaseRenderEngine):
         from src.engines.ffmpeg.ffmpeg_ui import FFMPEGUI
         return FFMPEGUI.get_options(self)
 
-    @classmethod
-    def supported_extensions(cls):
-        help_text = (subprocess.check_output([cls().renderer_path(), '-h', 'full'], stderr=subprocess.STDOUT)
-                     .decode('utf-8'))
+    def supported_extensions(self):
+        help_text = (subprocess.check_output([self.renderer_path(), '-h', 'full'], stderr=subprocess.STDOUT,
+                                             creationflags=_creationflags).decode('utf-8'))
         found = re.findall(r'extensions that .* is allowed to access \(default "(.*)"', help_text)
         found_extensions = set()
         for match in found:
@@ -34,8 +35,8 @@ class FFMPEG(BaseRenderEngine):
     def version(self):
         version = None
         try:
-            ver_out = subprocess.check_output([self.renderer_path(), '-version'],
-                                              timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
+            ver_out = subprocess.check_output([self.renderer_path(), '-version'], timeout=SUBPROCESS_TIMEOUT,
+                                              creationflags=_creationflags).decode('utf-8')
             match = re.match(r".*version\s*([\w.*]+)\W*", ver_out)
             if match:
                 version = match.groups()[0]
@@ -50,7 +51,8 @@ class FFMPEG(BaseRenderEngine):
                 'ffprobe', '-v', 'quiet', '-print_format', 'json',
                 '-show_streams', '-select_streams', 'v', project_path
             ]
-            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True,
+                                             creationflags=_creationflags)
             video_info = json.loads(output)
 
             # Extract the necessary information
@@ -81,7 +83,7 @@ class FFMPEG(BaseRenderEngine):
 
     def get_encoders(self):
         raw_stdout = subprocess.check_output([self.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL,
-                                             timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
+                                             timeout=SUBPROCESS_TIMEOUT, creationflags=_creationflags).decode('utf-8')
         pattern = r'(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
         encoders = [m.groupdict() for m in re.finditer(pattern, raw_stdout)]
         return encoders
@@ -93,7 +95,8 @@ class FFMPEG(BaseRenderEngine):
     def get_all_formats(self):
         try:
             formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
-                                                  timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
+                                                  timeout=SUBPROCESS_TIMEOUT,
+                                                  creationflags=_creationflags).decode('utf-8')
             pattern = r'(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)'
             all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
             return all_formats
@@ -105,7 +108,8 @@ class FFMPEG(BaseRenderEngine):
         # Extract the common extension using regex
         muxer_flag = 'muxer' if 'E' in ffmpeg_format['type'] else 'demuxer'
         format_detail_raw = subprocess.check_output(
-            [self.renderer_path(), '-hide_banner', '-h', f"{muxer_flag}={ffmpeg_format['id']}"]).decode('utf-8')
+            [self.renderer_path(), '-hide_banner', '-h', f"{muxer_flag}={ffmpeg_format['id']}"],
+            creationflags=_creationflags).decode('utf-8')
         pattern = r"Common extensions: (\w+)"
         common_extensions = re.findall(pattern, format_detail_raw)
         found_extensions = []
@@ -119,7 +123,7 @@ class FFMPEG(BaseRenderEngine):
     def get_frame_count(self, path_to_file):
         raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
                                               '-f', 'null', '-'], stderr=subprocess.STDOUT,
-                                             timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
+                                             timeout=SUBPROCESS_TIMEOUT, creationflags=_creationflags).decode('utf-8')
         match = re.findall(r'frame=\s*(\d+)', raw_stdout)
         if match:
             frame_number = int(match[-1])
@@ -127,8 +131,8 @@ class FFMPEG(BaseRenderEngine):
         return -1
 
     def get_arguments(self):
-        help_text = (subprocess.check_output([self.renderer_path(), '-h', 'long'], stderr=subprocess.STDOUT)
-                     .decode('utf-8'))
+        help_text = (subprocess.check_output([self.renderer_path(), '-h', 'long'], stderr=subprocess.STDOUT,
+                                             creationflags=_creationflags).decode('utf-8'))
         lines = help_text.splitlines()
 
         options = {}

src/engines/ffmpeg/ffmpeg_ui.py

@@ -1,5 +1,5 @@
 class FFMPEGUI:
     @staticmethod
-    def get_options(instance):
+    def get_options(system_info):
         options = []
         return options

src/init.py

@@ -1,11 +1,9 @@
-''' app/init.py '''
 import logging
 import multiprocessing
 import os
 import socket
 import sys
 import threading
-import time
 from collections import deque
 
 from src.api.api_server import start_server
@@ -22,8 +20,10 @@ logger = logging.getLogger()
 
 
 def run(server_only=False) -> int:
-    """
-    Initializes the application and runs it.
+    """Initializes the application and runs it.
+
+    Args:
+        server_only: Run in server-only CLI mode. Default is False (runs in GUI mode).
 
     Returns:
         int: The exit status code.
@@ -84,9 +84,7 @@ def run(server_only=False) -> int:
                                      'system_os_version': current_system_os_version()}
         ZeroconfServer.start()
         logger.info(f"Zordon Render Server started - Hostname: {local_hostname}")
-
-        RenderQueue.evaluation_inverval = Config.queue_eval_seconds
-        RenderQueue.start()
+        RenderQueue.start()  # Start evaluating the render queue
 
         # start in gui or server only (cli) mode
         logger.debug(f"Launching in {'server only' if server_only else 'GUI'} mode")
@@ -137,8 +135,8 @@ def __setup_buffer_handler():
 
     buffer_handler = BufferingHandler()
     buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
-    logger = logging.getLogger()
-    logger.addHandler(buffer_handler)
+    new_logger = logging.getLogger()
+    new_logger.addHandler(buffer_handler)
     return buffer_handler
 
 

src/render_queue.py

@@ -29,19 +29,37 @@ class RenderQueue:
     maximum_renderer_instances = {'blender': 1, 'aerender': 1, 'ffmpeg': 4}
     last_saved_counts = {}
     is_running = False
-    __eval_thread = None
-    evaluation_inverval = 1
 
     # --------------------------------------------
-    # Start / Stop Background Updates
+    # Render Queue Evaluation:
     # --------------------------------------------
 
     @classmethod
     def start(cls):
+        """Start evaluating the render queue"""
         logger.debug("Starting render queue updates")
         cls.is_running = True
         cls.evaluate_queue()
 
+    @classmethod
+    def evaluate_queue(cls):
+        try:
+            not_started = cls.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
+            for job in not_started:
+                if cls.is_available_for_job(job.renderer, job.priority):
+                    cls.start_job(job)
+
+            scheduled = cls.jobs_with_status(RenderStatus.SCHEDULED, priority_sorted=True)
+            for job in scheduled:
+                if job.scheduled_start <= datetime.now():
+                    logger.debug(f"Starting scheduled job: {job}")
+                    cls.start_job(job)
+
+            if cls.last_saved_counts != cls.job_counts():
+                cls.save_state()
+        except DetachedInstanceError:
+            pass
+
     @classmethod
     def __local_job_status_changed(cls, job_id, old_status, new_status):
         render_job = RenderQueue.job_with_id(job_id, none_ok=True)
@@ -55,20 +73,9 @@ class RenderQueue:
         cls.is_running = False
 
     # --------------------------------------------
-    # Queue Management
+    # Fetch Jobs:
     # --------------------------------------------
 
-    @classmethod
-    def add_to_render_queue(cls, render_job, force_start=False):
-        logger.info(f"Adding job to render queue: {render_job}")
-        cls.job_queue.append(render_job)
-        if cls.is_running and force_start and render_job.status in (RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED):
-            cls.start_job(render_job)
-        cls.session.add(render_job)
-        cls.save_state()
-        if cls.is_running:
-            cls.evaluate_queue()
-
     @classmethod
     def all_jobs(cls):
         return cls.job_queue
@@ -98,12 +105,15 @@ class RenderQueue:
         return found_job
 
     @classmethod
-    def clear_history(cls):
-        to_remove = [x for x in cls.all_jobs() if x.status in [RenderStatus.CANCELLED,
-                                                               RenderStatus.COMPLETED, RenderStatus.ERROR]]
-        for job_to_remove in to_remove:
-            cls.delete_job(job_to_remove)
-        cls.save_state()
+    def job_counts(cls):
+        job_counts = {}
+        for job_status in RenderStatus:
+            job_counts[job_status.value] = len(cls.jobs_with_status(job_status))
+        return job_counts
+
+    # --------------------------------------------
+    # Startup / Shutdown:
+    # --------------------------------------------
 
     @classmethod
     def load_state(cls, database_directory):
@@ -128,6 +138,16 @@ class RenderQueue:
         cls.save_state()
         cls.session.close()
 
+    # --------------------------------------------
+    # Renderer Availability:
+    # --------------------------------------------
+
+    @classmethod
+    def renderer_instances(cls):
+        from collections import Counter
+        all_instances = [x.renderer for x in cls.running_jobs()]
+        return Counter(all_instances)
+
     @classmethod
     def is_available_for_job(cls, renderer, priority=2):
 
@@ -137,24 +157,20 @@ class RenderQueue:
         maxed_out_instances = renderer in instances.keys() and instances[renderer] >= max_allowed_instances
         return not maxed_out_instances and not higher_priority_jobs
 
+    # --------------------------------------------
+    # Job Lifecycle Management:
+    # --------------------------------------------
+
     @classmethod
-    def evaluate_queue(cls):
-        try:
-            not_started = cls.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
-            for job in not_started:
-                if cls.is_available_for_job(job.renderer, job.priority):
-                    cls.start_job(job)
-
-            scheduled = cls.jobs_with_status(RenderStatus.SCHEDULED, priority_sorted=True)
-            for job in scheduled:
-                if job.scheduled_start <= datetime.now():
-                    logger.debug(f"Starting scheduled job: {job}")
-                    cls.start_job(job)
-
-            if cls.last_saved_counts != cls.job_counts():
-                cls.save_state()
-        except DetachedInstanceError:
-            pass
+    def add_to_render_queue(cls, render_job, force_start=False):
+        logger.info(f"Adding job to render queue: {render_job}")
+        cls.job_queue.append(render_job)
+        if cls.is_running and force_start and render_job.status in (RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED):
+            cls.start_job(render_job)
+        cls.session.add(render_job)
+        cls.save_state()
+        if cls.is_running:
+            cls.evaluate_queue()
 
     @classmethod
     def start_job(cls, job):
@@ -177,15 +193,14 @@ class RenderQueue:
         cls.save_state()
         return True
 
-    @classmethod
-    def renderer_instances(cls):
-        from collections import Counter
-        all_instances = [x.renderer for x in cls.running_jobs()]
-        return Counter(all_instances)
+    # --------------------------------------------
+    # Miscellaneous:
+    # --------------------------------------------
 
     @classmethod
-    def job_counts(cls):
-        job_counts = {}
-        for job_status in RenderStatus:
-            job_counts[job_status.value] = len(cls.jobs_with_status(job_status))
-        return job_counts
+    def clear_history(cls):
+        to_remove = [x for x in cls.all_jobs() if x.status in [RenderStatus.CANCELLED,
+                                                               RenderStatus.COMPLETED, RenderStatus.ERROR]]
+        for job_to_remove in to_remove:
+            cls.delete_job(job_to_remove)
+        cls.save_state()

src/ui/about_window.py

@@ -0,0 +1,74 @@
+import os
+import sys
+
+from PyQt6.QtCore import Qt
+from PyQt6.QtGui import QPixmap
+from PyQt6.QtWidgets import QDialog, QVBoxLayout, QLabel, QDialogButtonBox, QHBoxLayout
+
+from version import *
+
+
+class AboutDialog(QDialog):
+    def __init__(self):
+        super().__init__()
+        self.setWindowTitle(f"About {APP_NAME}")
+
+        # Create the layout
+        layout = QVBoxLayout()
+
+        # App Icon
+        icon_name = 'Server.png'  # todo: temp icon - replace with final later
+        icon_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+                                 'resources', icon_name)
+        icon_label = QLabel(self)
+        icon_pixmap = QPixmap(icon_path)
+        icon_label.setPixmap(icon_pixmap)
+        icon_layout = QHBoxLayout()
+        icon_layout.addStretch()
+        icon_layout.addWidget(icon_label)
+        icon_layout.addStretch()
+        layout.addLayout(icon_layout)
+
+        # Application name
+        name_label = QLabel(f"<h2>{APP_NAME}</h2>")
+        layout.addWidget(name_label)
+
+        # Description
+        description_label = QLabel(APP_DESCRIPTION)
+        layout.addWidget(description_label)
+
+        # Version
+        version_label = QLabel(f"<strong>Version:</strong> {APP_VERSION}")
+        layout.addWidget(version_label)
+
+        # Contributors
+        contributors_label = QLabel(f"Copyright © {APP_COPYRIGHT_YEAR} {APP_AUTHOR}")
+        layout.addWidget(contributors_label)
+
+        # License
+        license_label = QLabel(f"Released under {APP_LICENSE}")
+        layout.addWidget(license_label)
+
+        # Add an "OK" button to close the dialog
+        button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok)
+        button_box.accepted.connect(self.accept)
+        layout.addWidget(button_box)
+
+        # Set the layout for the dialog
+        self.setLayout(layout)
+
+        # Make the dialog non-resizable
+        self.setWindowFlags(self.windowFlags() & ~Qt.WindowType.WindowContextHelpButtonHint)
+        self.setFixedSize(self.sizeHint())
+
+
+if __name__ == '__main__':
+    # lazy load GUI frameworks
+    from PyQt6.QtWidgets import QApplication
+
+    # load application
+    app: QApplication = QApplication(sys.argv)
+    window: AboutDialog = AboutDialog()
+    window.show()
+
+    app.exec()

src/ui/add_job.py

@@ -68,11 +68,12 @@ class NewRenderJobForm(QWidget):
         # Setup
         self.setWindowTitle("New Job")
         self.setup_ui()
+        self.update_renderer_info()
         self.setup_project()
 
         # get renderer info in bg thread
-        t = threading.Thread(target=self.update_renderer_info)
-        t.start()
+        # t = threading.Thread(target=self.update_renderer_info)
+        # t.start()
 
         self.show()
 
@@ -348,7 +349,8 @@ class NewRenderJobForm(QWidget):
             # Dynamic Engine Options
             clear_layout(self.renderer_options_layout)  # clear old options
             # dynamically populate option list
-            self.current_engine_options = engine().ui_options()
+            system_info = self.renderer_info.get(engine.name(), {}).get('system_info', {})
+            self.current_engine_options = engine.ui_options(system_info=system_info)
             for option in self.current_engine_options:
                 h_layout = QHBoxLayout()
                 label = QLabel(option['name'].replace('_', ' ').capitalize() + ':')
@@ -375,7 +377,7 @@ class NewRenderJobForm(QWidget):
             self.cameras_group.setHidden(True)
         self.submit_button.setEnabled(enabled)
 
-    def after_job_submission(self, result):
+    def after_job_submission(self, error_string):
 
         # UI cleanup
         self.submit_progress.setMaximum(0)
@@ -387,7 +389,7 @@ class NewRenderJobForm(QWidget):
         self.toggle_renderer_enablement(True)
 
         self.msg_box = QMessageBox()
-        if result.ok:
+        if not error_string:
             self.msg_box.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
             self.msg_box.setIcon(QMessageBox.Icon.Information)
             self.msg_box.setText("Job successfully submitted to server. Submit another?")
@@ -398,7 +400,7 @@ class NewRenderJobForm(QWidget):
         else:
             self.msg_box.setStandardButtons(QMessageBox.StandardButton.Ok)
             self.msg_box.setIcon(QMessageBox.Icon.Critical)
-            self.msg_box.setText(result.text or "Unknown error")
+            self.msg_box.setText(error_string)
             self.msg_box.setWindowTitle("Error")
             self.msg_box.exec()
 
@@ -429,7 +431,7 @@ class NewRenderJobForm(QWidget):
 class SubmitWorker(QThread):
     """Worker class called to submit all the jobs to the server and update the UI accordingly"""
 
-    message_signal = pyqtSignal(Response)
+    message_signal = pyqtSignal(str)
     update_ui_signal = pyqtSignal(str, str)
 
     def __init__(self, window):
@@ -445,61 +447,69 @@ class SubmitWorker(QThread):
                 self.update_ui_signal.emit(hostname, percent)
             return callback
 
-        hostname = self.window.server_input.currentText()
-        job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(),
-                    'renderer': self.window.renderer_type.currentText().lower(),
-                    'engine_version': self.window.renderer_version_combo.currentText(),
-                    'args': {'raw': self.window.raw_args.text(),
-                             'export_format': self.window.file_format_combo.currentText()},
-                    'output_path': self.window.render_name_input.text(),
-                    'start_frame': self.window.start_frame_input.value(),
-                    'end_frame': self.window.end_frame_input.value(),
-                    'priority': self.window.priority_input.currentIndex() + 1,
-                    'notes': self.window.notes_input.toPlainText(),
-                    'enable_split_jobs': self.window.enable_splitjobs.isChecked(),
-                    'split_jobs_same_os': self.window.splitjobs_same_os.isChecked(),
-                    'name': self.window.render_name_input.text()}
+        try:
+            hostname = self.window.server_input.currentText()
+            job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(),
+                        'renderer': self.window.renderer_type.currentText().lower(),
+                        'engine_version': self.window.renderer_version_combo.currentText(),
+                        'args': {'raw': self.window.raw_args.text(),
+                                 'export_format': self.window.file_format_combo.currentText()},
+                        'output_path': self.window.render_name_input.text(),
+                        'start_frame': self.window.start_frame_input.value(),
+                        'end_frame': self.window.end_frame_input.value(),
+                        'priority': self.window.priority_input.currentIndex() + 1,
+                        'notes': self.window.notes_input.toPlainText(),
+                        'enable_split_jobs': self.window.enable_splitjobs.isChecked(),
+                        'split_jobs_same_os': self.window.splitjobs_same_os.isChecked(),
+                        'name': self.window.render_name_input.text()}
 
-        # get the dynamic args
-        for i in range(self.window.renderer_options_layout.count()):
-            item = self.window.renderer_options_layout.itemAt(i)
-            layout = item.layout()  # get the layout
-            for x in range(layout.count()):
-                z = layout.itemAt(x)
-                widget = z.widget()
-                if isinstance(widget, QComboBox):
-                    job_json['args'][self.window.current_engine_options[i]['name']] = widget.currentText()
-                elif isinstance(widget, QLineEdit):
-                    job_json['args'][self.window.current_engine_options[i]['name']] = widget.text()
+            # get the dynamic args
+            for i in range(self.window.renderer_options_layout.count()):
+                item = self.window.renderer_options_layout.itemAt(i)
+                layout = item.layout()  # get the layout
+                for x in range(layout.count()):
+                    z = layout.itemAt(x)
+                    widget = z.widget()
+                    if isinstance(widget, QComboBox):
+                        job_json['args'][self.window.current_engine_options[i]['name']] = widget.currentText()
+                    elif isinstance(widget, QLineEdit):
+                        job_json['args'][self.window.current_engine_options[i]['name']] = widget.text()
 
-        # determine if any cameras are checked
-        selected_cameras = []
-        if self.window.cameras_list.count() and not self.window.cameras_group.isHidden():
-            for index in range(self.window.cameras_list.count()):
-                item = self.window.cameras_list.item(index)
-                if item.checkState() == Qt.CheckState.Checked:
-                    selected_cameras.append(item.text().rsplit('-', 1)[0].strip())  # cleanup to just camera name
+            # determine if any cameras are checked
+            selected_cameras = []
+            if self.window.cameras_list.count() and not self.window.cameras_group.isHidden():
+                for index in range(self.window.cameras_list.count()):
+                    item = self.window.cameras_list.item(index)
+                    if item.checkState() == Qt.CheckState.Checked:
+                        selected_cameras.append(item.text().rsplit('-', 1)[0].strip())  # cleanup to just camera name
 
-        # process cameras into nested format
-        input_path = self.window.scene_file_input.text()
-        if selected_cameras:
-            job_list = []
-            for cam in selected_cameras:
-                job_copy = copy.deepcopy(job_json)
-                job_copy['args']['camera'] = cam
-                job_copy['name'] = job_copy['name'].replace(' ', '-') + "_" + cam.replace(' ', '')
-                job_copy['output_path'] = job_copy['name']
-                job_list.append(job_copy)
-        else:
-            job_list = [job_json]
+            # process cameras into nested format
+            input_path = self.window.scene_file_input.text()
+            if selected_cameras:
+                job_list = []
+                for cam in selected_cameras:
+                    job_copy = copy.deepcopy(job_json)
+                    job_copy['args']['camera'] = cam
+                    job_copy['name'] = job_copy['name'].replace(' ', '-') + "_" + cam.replace(' ', '')
+                    job_copy['output_path'] = job_copy['name']
+                    job_list.append(job_copy)
+            else:
+                job_list = [job_json]
 
-        # presubmission tasks
-        engine = EngineManager.engine_with_name(self.window.renderer_type.currentText().lower())
-        input_path = engine().perform_presubmission_tasks(input_path)
-        # submit
-        result = self.window.server_proxy.post_job_to_server(file_path=input_path, job_list=job_list,
-                                                      callback=create_callback)
-        self.message_signal.emit(result)
+            # presubmission tasks
+            engine = EngineManager.engine_with_name(self.window.renderer_type.currentText().lower())
+            input_path = engine().perform_presubmission_tasks(input_path)
+            # submit
+            err_msg = ""
+            result = self.window.server_proxy.post_job_to_server(file_path=input_path, job_list=job_list,
+                                                          callback=create_callback)
+            if not (result and result.ok):
+                err_msg = "Error posting job to server."
+
+            self.message_signal.emit(err_msg)
+
+        except Exception as e:
+            self.message_signal.emit(str(e))
 
 
 class GetProjectInfoWorker(QThread):

src/ui/main_window.py

@@ -20,14 +20,14 @@ from src.render_queue import RenderQueue
 from src.utilities.misc_helper import get_time_elapsed, resources_dir, is_localhost
 from src.utilities.status_utils import RenderStatus
 from src.utilities.zeroconf_server import ZeroconfServer
-from .add_job import NewRenderJobForm
-from .console import ConsoleWindow
-from .engine_browser import EngineBrowserWindow
-from .log_viewer import LogViewer
-from .widgets.menubar import MenuBar
-from .widgets.proportional_image_label import ProportionalImageLabel
-from .widgets.statusbar import StatusBar
-from .widgets.toolbar import ToolBar
+from src.ui.add_job import NewRenderJobForm
+from src.ui.console import ConsoleWindow
+from src.ui.engine_browser import EngineBrowserWindow
+from src.ui.log_viewer import LogViewer
+from src.ui.widgets.menubar import MenuBar
+from src.ui.widgets.proportional_image_label import ProportionalImageLabel
+from src.ui.widgets.statusbar import StatusBar
+from src.ui.widgets.toolbar import ToolBar
 from src.api.serverproxy_manager import ServerProxyManager
 from src.utilities.misc_helper import launch_url
 
@@ -325,7 +325,7 @@ class MainWindow(QMainWindow):
             current_status = self.job_list_view.item(selected_row.row(), 4).text()
 
             # show / hide the stop button
-            show_stop_button = current_status.lower() == 'running'
+            show_stop_button = "%" in current_status
             self.topbar.actions_call['Stop Job'].setEnabled(show_stop_button)
             self.topbar.actions_call['Stop Job'].setVisible(show_stop_button)
             self.topbar.actions_call['Delete Job'].setEnabled(not show_stop_button)
@@ -512,7 +512,7 @@ class MainWindow(QMainWindow):
 
     def stop_job(self, event):
         """
-        Event handler for the "Exit" button. Closes the application.
+        Event handler for the Stop Job button
         """
         job_ids = self.selected_job_ids()
         if not job_ids:
@@ -522,14 +522,14 @@ class MainWindow(QMainWindow):
             job = next((job for job in self.current_server_proxy.get_all_jobs() if job.get('id') == job_ids[0]), None)
             if job:
                 display_name = job.get('name', os.path.basename(job.get('input_path', '')))
-                message = f"Are you sure you want to delete the job:\n{display_name}?"
+                message = f"Are you sure you want to stop the job:\n{display_name}?"
             else:
                 return  # Job not found, handle this case as needed
         else:
-            message = f"Are you sure you want to delete these {len(job_ids)} jobs?"
+            message = f"Are you sure you want to stop these {len(job_ids)} jobs?"
 
         # Display the message box and check the response in one go
-        msg_box = QMessageBox(QMessageBox.Icon.Warning, "Delete Job", message,
+        msg_box = QMessageBox(QMessageBox.Icon.Warning, "Stop Job", message,
                               QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, self)
 
         if msg_box.exec() == QMessageBox.StandardButton.Yes:
@@ -565,7 +565,15 @@ class MainWindow(QMainWindow):
             self.fetch_jobs(clear_table=True)
 
     def download_files(self, event):
-        pass
+
+        job_ids = self.selected_job_ids()
+        if not job_ids:
+            return
+
+        import webbrowser
+        download_url = (f"http://{self.current_server_proxy.hostname}:{self.current_server_proxy.port}"
+                        f"/api/job/{job_ids[0]}/download_all")
+        webbrowser.open(download_url)
 
     def open_files(self, event):
         job_ids = self.selected_job_ids()

src/ui/widgets/menubar.py

@@ -1,5 +1,8 @@
 ''' app/ui/widgets/menubar.py '''
-from PyQt6.QtWidgets import QMenuBar
+import sys
+
+from PyQt6.QtGui import QAction
+from PyQt6.QtWidgets import QMenuBar, QApplication
 
 
 class MenuBar(QMenuBar):
@@ -12,12 +15,43 @@ class MenuBar(QMenuBar):
 
     def __init__(self, parent=None) -> None:
         super().__init__(parent)
+
+        # setup menus
         file_menu = self.addMenu("File")
         # edit_menu = self.addMenu("Edit")
         # view_menu = self.addMenu("View")
-        # help_menu = self.addMenu("Help")
+        help_menu = self.addMenu("Help")
 
-        # Add actions to the menus
-        # file_menu.addAction(self.parent().topbar.actions_call["Open"]) # type: ignore
-        # file_menu.addAction(self.parent().topbar.actions_call["Save"]) # type: ignore
-        # file_menu.addAction(self.parent().topbar.actions_call["Exit"]) # type: ignore
+        # --file menu--
+        # new job
+        new_job_action = QAction("New Job...", self)
+        new_job_action.setShortcut(f'Ctrl+N')
+        new_job_action.triggered.connect(self.new_job)
+        file_menu.addAction(new_job_action)
+        # settings
+        settings_action = QAction("Settings...", self)
+        settings_action.triggered.connect(self.show_settings)
+        settings_action.setShortcut(f'Ctrl+,')
+        # file_menu.addAction(settings_action)  # todo: enable once we have a setting screen
+        # exit
+        exit_action = QAction('&Exit', self)
+        exit_action.setShortcut('Ctrl+Q')
+        exit_action.triggered.connect(QApplication.instance().quit)
+        file_menu.addAction(exit_action)
+
+        # --help menu--
+        about_action = QAction("About", self)
+        about_action.triggered.connect(self.show_about)
+        help_menu.addAction(about_action)
+
+    def new_job(self):
+        self.parent().new_job()
+
+    def show_settings(self):
+        pass
+
+    @staticmethod
+    def show_about():
+        from src.ui.about_window import AboutDialog
+        dialog = AboutDialog()
+        dialog.exec()

src/utilities/server_helper.py

@@ -1,47 +1,200 @@
 import logging
 import os
-import subprocess
-import threading
+import zipfile
+from concurrent.futures import ThreadPoolExecutor
 
-from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
+import requests
+
+from src.api.server_proxy import RenderServerProxy
+from src.utilities.misc_helper import get_file_size_human
+from src.utilities.zeroconf_server import ZeroconfServer
 
 logger = logging.getLogger()
 
 
-def generate_thumbnail_for_job(job, thumb_video_path, thumb_image_path, max_width=320):
+def download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname):
+    success = True
+    try:
+        local_files = [os.path.basename(x) for x in local_job.file_list()]
+        subjob_proxy = RenderServerProxy(subjob_hostname)
+        subjob_files = subjob_proxy.get_job_files_list(job_id=subjob_id) or []
 
-    # Simple thread to generate thumbs in background
-    def generate_thumb_thread(source):
-        in_progress_path = thumb_video_path + '_IN-PROGRESS'
-        subprocess.run(['touch', in_progress_path])
-        try:
-            logger.debug(f"Generating video thumbnail for {source}")
-            generate_thumbnail(source_path=source, dest_path=thumb_video_path, max_width=max_width)
-        except subprocess.CalledProcessError as err:
-            logger.error(f"Error generating video thumbnail for {source}: {err}")
+        for subjob_filename in subjob_files:
+            if subjob_filename not in local_files:
+                try:
+                    logger.debug(f"Downloading new file '{subjob_filename}' from {subjob_hostname}")
+                    local_save_path = os.path.join(os.path.dirname(local_job.output_path), subjob_filename)
+                    subjob_proxy.download_job_file(job_id=subjob_id, job_filename=subjob_filename,
+                                                   save_path=local_save_path)
+                    logger.debug(f'Downloaded successfully - {local_save_path}')
+                except Exception as e:
+                    logger.error(f"Error downloading file '{subjob_filename}' from {subjob_hostname}: {e}")
+                    success = False
+    except Exception as e:
+        logger.exception(f'Uncaught exception while trying to download from subjob: {e}')
+        success = False
+    return success
 
-        try:
-            os.remove(in_progress_path)
-        except FileNotFoundError:
-            pass
 
-    # Determine best source file to use for thumbs
-    source_files = job.file_list() or [job.input_path]
-    if source_files:
-        video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', 'mkv']
-        image_formats = ['.jpg', '.png', '.exr']
+def download_all_from_subjob(local_job, subjob_id, subjob_hostname):
+    """
+    Downloads and extracts files from a completed subjob on a remote server.
 
-        image_files = [f for f in source_files if os.path.splitext(f)[-1].lower() in image_formats]
-        video_files = [f for f in source_files if os.path.splitext(f)[-1].lower() in video_formats]
+    Parameters:
+    local_job (BaseRenderWorker): The local parent job worker.
+    subjob_id (str or int): The ID of the subjob.
+    subjob_hostname (str): The hostname of the remote server where the subjob is located.
 
-        if (video_files or image_files) and not os.path.exists(thumb_image_path):
+    Returns:
+    bool: True if the files have been downloaded and extracted successfully, False otherwise.
+    """
+
+    child_key = f'{subjob_id}@{subjob_hostname}'
+    logname = f"{local_job.id}:{child_key}"
+    zip_file_path = local_job.output_path + f'_{subjob_hostname}_{subjob_id}.zip'
+
+    # download zip file from server
+    try:
+        local_job.children[child_key]['download_status'] = 'working'
+        logger.info(f"Downloading completed subjob files from {subjob_hostname} to localhost")
+        RenderServerProxy(subjob_hostname).download_all_job_files(subjob_id, zip_file_path)
+        logger.info(f"File transfer complete for {logname} - Transferred {get_file_size_human(zip_file_path)}")
+    except Exception as e:
+        logger.error(f"Error downloading files from remote server: {e}")
+        local_job.children[child_key]['download_status'] = 'failed'
+        return False
+
+    # extract zip
+    try:
+        logger.debug(f"Extracting zip file: {zip_file_path}")
+        extract_path = os.path.dirname(zip_file_path)
+        with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
+            zip_ref.extractall(extract_path)
+            logger.info(f"Successfully extracted zip to: {extract_path}")
+        os.remove(zip_file_path)
+        local_job.children[child_key]['download_status'] = 'complete'
+    except Exception as e:
+        logger.exception(f"Exception extracting zip file: {e}")
+        local_job.children[child_key]['download_status'] = 'failed'
+
+    return local_job.children[child_key].get('download_status', None) == 'complete'
+
+
+def distribute_server_work(start_frame, end_frame, available_servers, method='evenly'):
+    """
+    Splits the frame range among available servers proportionally based on their performance (CPU count).
+
+    Args:
+        start_frame (int): The start frame number of the animation to be rendered.
+        end_frame (int): The end frame number of the animation to be rendered.
+        available_servers (list): A list of available server dictionaries. Each server dictionary should include
+        'hostname' and 'cpu_count' keys (see find_available_servers).
+        method (str, optional): Specifies the distribution method. Possible values are 'cpu_benchmark', 'cpu_count'
+        and 'evenly'.
+        Defaults to 'cpu_benchmark'.
+
+    Returns:
+        list: A list of server dictionaries where each dictionary includes the frame range and total number of
+        frames to be rendered by the server.
+    """
+    # Calculate respective frames for each server
+    def divide_frames_by_cpu_count(frame_start, frame_end, servers):
+        total_frames = frame_end - frame_start + 1
+        total_cpus = sum(server['cpu_count'] for server in servers)
+
+        frame_ranges = {}
+        current_frame = frame_start
+        allocated_frames = 0
+
+        for i, server in enumerate(servers):
+            if i == len(servers) - 1:  # if it's the last server
+                # Give all remaining frames to the last server
+                num_frames = total_frames - allocated_frames
+            else:
+                num_frames = round((server['cpu_count'] / total_cpus) * total_frames)
+                allocated_frames += num_frames
+
+            frame_end_for_server = current_frame + num_frames - 1
+
+            if current_frame <= frame_end_for_server:
+                frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
+            current_frame = frame_end_for_server + 1
+
+        return frame_ranges
+
+    def divide_frames_by_benchmark(frame_start, frame_end, servers):
+
+        def fetch_benchmark(server):
             try:
-                path_of_source = image_files[0] if image_files else video_files[0]
-                logger.debug(f"Generating image thumbnail for {path_of_source}")
-                save_first_frame(source_path=path_of_source, dest_path=thumb_image_path, max_width=max_width)
-            except Exception as e:
-                logger.error(f"Exception saving first frame: {e}")
+                benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}'
+                                         f'/api/cpu_benchmark').text
+                server['cpu_benchmark'] = benchmark
+                logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}')
+            except requests.exceptions.RequestException as e:
+                logger.error(f'Error fetching benchmark for {server["hostname"]}: {e}')
 
-        if video_files and not os.path.exists(thumb_video_path):
-            x = threading.Thread(target=generate_thumb_thread, args=(video_files[0],))
-            x.start()
+        # Number of threads to use (can adjust based on your needs or number of servers)
+        threads = len(servers)
+
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            executor.map(fetch_benchmark, servers)
+
+        total_frames = frame_end - frame_start + 1
+        total_performance = sum(int(server['cpu_benchmark']) for server in servers)
+
+        frame_ranges = {}
+        current_frame = frame_start
+        allocated_frames = 0
+
+        for i, server in enumerate(servers):
+            if i == len(servers) - 1:  # if it's the last server
+                # Give all remaining frames to the last server
+                num_frames = total_frames - allocated_frames
+            else:
+                num_frames = round((int(server['cpu_benchmark']) / total_performance) * total_frames)
+                allocated_frames += num_frames
+
+            frame_end_for_server = current_frame + num_frames - 1
+
+            if current_frame <= frame_end_for_server:
+                frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
+            current_frame = frame_end_for_server + 1
+
+        return frame_ranges
+
+    def divide_frames_equally(frame_start, frame_end, servers):
+        frame_range = frame_end - frame_start + 1
+        frames_per_server = frame_range // len(servers)
+        leftover_frames = frame_range % len(servers)
+
+        frame_ranges = {}
+        current_start = frame_start
+        for i, server in enumerate(servers):
+            current_end = current_start + frames_per_server - 1
+            if leftover_frames > 0:
+                current_end += 1
+                leftover_frames -= 1
+            if current_start <= current_end:
+                frame_ranges[server['hostname']] = (current_start, current_end)
+            current_start = current_end + 1
+
+        return frame_ranges
+
+    if len(available_servers) == 1:
+        breakdown = {available_servers[0]['hostname']: (start_frame, end_frame)}
+    else:
+        logger.debug(f'Splitting between {len(available_servers)} servers by {method} method')
+        if method == 'evenly':
+            breakdown = divide_frames_equally(start_frame, end_frame, available_servers)
+        elif method == 'cpu_benchmark':
+            breakdown = divide_frames_by_benchmark(start_frame, end_frame, available_servers)
+        elif method == 'cpu_count':
+            breakdown = divide_frames_by_cpu_count(start_frame, end_frame, available_servers)
+        else:
+            raise ValueError(f"Invalid distribution method: {method}")
+
+    server_breakdown = [server for server in available_servers if breakdown.get(server['hostname']) is not None]
+    for server in server_breakdown:
+        server['frame_range'] = breakdown[server['hostname']]
+        server['total_frames'] = breakdown[server['hostname']][-1] - breakdown[server['hostname']][0] + 1
+    return server_breakdown

version.py

@@ -0,0 +1,6 @@
+APP_NAME = "Zordon"
+APP_VERSION = "0.0.1"
+APP_AUTHOR = "Brett Williams"
+APP_DESCRIPTION = "Distributed Render Farm Tools"
+APP_COPYRIGHT_YEAR = "2024"
+APP_LICENSE = "MIT License"