Engine and downloader refactoring (#50)

* Make downloaders subclass of base_downloader.py

* Link engines and downloaders together for all engines

* Replace / merge worker_factory.py with engine_manager.py

src/api/add_job_helpers.py

@@ -11,7 +11,7 @@ from tqdm import tqdm
 from werkzeug.utils import secure_filename
 
 from src.distributed_job_manager import DistributedJobManager
-from src.engines.core.worker_factory import RenderWorkerFactory
+from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue
 
 logger = logging.getLogger()
@@ -149,11 +149,11 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir, enable_spl
             logger.debug(f"New job output path: {output_path}")
 
             # create & configure jobs
-            worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'],
+            worker = EngineManager.create_worker(renderer=job_data['renderer'],
-                                                       input_path=loaded_project_local_path,
+                                                 input_path=loaded_project_local_path,
-                                                       output_path=output_path,
+                                                 output_path=output_path,
-                                                       engine_version=job_data.get('engine_version'),
+                                                 engine_version=job_data.get('engine_version'),
-                                                       args=job_data.get('args', {}))
+                                                 args=job_data.get('args', {}))
             worker.status = job_data.get("initial_status", worker.status)
             worker.parent = job_data.get("parent", worker.parent)
             worker.name = job_data.get("name", worker.name)

src/api/api_server.py

@@ -21,7 +21,6 @@ from src.api.server_proxy import RenderServerProxy
 from src.api.add_job_helpers import handle_uploaded_project_files, process_zipped_project, create_render_jobs
 from src.distributed_job_manager import DistributedJobManager
 from src.engines.core.base_worker import string_to_status, RenderStatus
-from src.engines.core.worker_factory import RenderWorkerFactory
 from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue, JobNotFoundError
 from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, current_system_os_version
@@ -390,7 +389,7 @@ def clear_history():
 @server.route('/api/status')
 def status():
     renderer_data = {}
-    for render_class in RenderWorkerFactory.supported_classes():
+    for render_class in EngineManager.supported_engines():
         if EngineManager.all_versions_for_engine(render_class.name):  # only return renderers installed on host
             renderer_data[render_class.engine.name()] = \
                 {'versions': EngineManager.all_versions_for_engine(render_class.engine.name()),
@@ -418,8 +417,8 @@ def status():
 @server.get('/api/renderer_info')
 def renderer_info():
     renderer_data = {}
-    for engine_name in RenderWorkerFactory.supported_renderers():
+    for engine_name in EngineManager.supported_engines():
-        engine = RenderWorkerFactory.class_for_name(engine_name).engine
+        engine = EngineManager.engine_with_name(engine_name)
 
         # Get all installed versions of engine
         installed_versions = EngineManager.all_versions_for_engine(engine_name)
@@ -482,7 +481,7 @@ def delete_engine_download():
 @server.get('/api/renderer/<renderer>/args')
 def get_renderer_args(renderer):
     try:
-        renderer_engine_class = RenderWorkerFactory.class_for_name(renderer).engine()
+        renderer_engine_class = EngineManager.engine_with_name(renderer)
         return renderer_engine_class.get_arguments()
     except LookupError:
         return f"Cannot find renderer '{renderer}'", 400
@@ -490,7 +489,7 @@ def get_renderer_args(renderer):
 
 @server.route('/upload')
 def upload_file_page():
-    return render_template('upload.html', supported_renderers=RenderWorkerFactory.supported_renderers())
+    return render_template('upload.html', supported_renderers=EngineManager.supported_engines())
 
 
 def start_server(background_thread=False):

src/engines/blender/blender_downloader.py

@@ -3,7 +3,8 @@ import re
 
 import requests
 
-from src.engines.core.downloader_core import download_and_extract_app
+from src.engines.blender.blender_engine import Blender
+from src.engines.core.base_downloader import EngineDownloader
 from src.utilities.misc_helper import current_system_os, current_system_cpu
 
 # url = "https://download.blender.org/release/"
@@ -13,10 +14,12 @@ logger = logging.getLogger()
 supported_formats = ['.zip', '.tar.xz', '.dmg']
 
 
-class BlenderDownloader:
+class BlenderDownloader(EngineDownloader):
+
+    engine = Blender
 
     @staticmethod
-    def get_major_versions():
+    def __get_major_versions():
         try:
             response = requests.get(url, timeout=5)
             response.raise_for_status()
@@ -30,10 +33,10 @@ class BlenderDownloader:
             return major_versions
         except requests.exceptions.RequestException as e:
             logger.error(f"Error: {e}")
-        return None
+        return []
 
     @staticmethod
-    def get_minor_versions(major_version, system_os=None, cpu=None):
+    def __get_minor_versions(major_version, system_os=None, cpu=None):
 
         try:
             base_url = url + 'Blender' + major_version
@@ -63,17 +66,8 @@ class BlenderDownloader:
             logger.exception(e)
         return []
 
-    @classmethod
-    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
-        requested_major_version = '.'.join(version.split('.')[:2])
-        minor_versions = cls.get_minor_versions(requested_major_version, system_os, cpu)
-        for minor in minor_versions:
-            if minor['version'] == version:
-                return minor
-        return None
-
     @staticmethod
-    def find_LTS_versions():
+    def __find_LTS_versions():
         response = requests.get('https://www.blender.org/download/lts/')
         response.raise_for_status()
 
@@ -87,11 +81,21 @@ class BlenderDownloader:
     @classmethod
     def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
         try:
-            major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0]
+            major_version = cls.__find_LTS_versions()[0] if lts_only else cls.__get_major_versions()[0]
-            most_recent = cls.get_minor_versions(major_version=major_version, system_os=system_os, cpu=cpu)
+            most_recent = cls.__get_minor_versions(major_version=major_version, system_os=system_os, cpu=cpu)
             return most_recent[0]
-        except IndexError:
+        except (IndexError, requests.exceptions.RequestException):
-            logger.error("Cannot find a most recent version")
+            logger.error(f"Cannot get most recent version of blender")
+        return {}
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        requested_major_version = '.'.join(version.split('.')[:2])
+        minor_versions = cls.__get_minor_versions(requested_major_version, system_os, cpu)
+        for minor in minor_versions:
+            if minor['version'] == version:
+                return minor
+        return None
 
     @classmethod
     def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
@@ -101,11 +105,11 @@ class BlenderDownloader:
         try:
             logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
             major_version = '.'.join(version.split('.')[:2])
-            minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
+            minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
             # we get the URL instead of calculating it ourselves. May change this
 
-            download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
+            cls.__download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
-                                     timeout=timeout)
+                                           timeout=timeout)
         except IndexError:
             logger.error("Cannot find requested engine")
 
@@ -113,5 +117,5 @@ class BlenderDownloader:
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
-    print(BlenderDownloader.get_major_versions())
+    print(BlenderDownloader.__get_major_versions())
 

src/engines/blender/blender_engine.py

@@ -13,6 +13,16 @@ class Blender(BaseRenderEngine):
     supported_extensions = ['.blend']
     binary_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender'}
 
+    @staticmethod
+    def downloader():
+        from src.engines.blender.blender_downloader import BlenderDownloader
+        return BlenderDownloader
+
+    @staticmethod
+    def worker_class():
+        from src.engines.blender.blender_worker import BlenderRenderWorker
+        return BlenderRenderWorker
+
     def version(self):
         version = None
         try:

src/engines/core/base_downloader.py

@@ -0,0 +1,158 @@
+import logging
+import os
+import shutil
+import tarfile
+import tempfile
+import zipfile
+
+import requests
+from tqdm import tqdm
+
+logger = logging.getLogger()
+
+
+class EngineDownloader:
+
+    supported_formats = ['.zip', '.tar.xz', '.dmg']
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
+        raise NotImplementedError  # implement this method in your engine subclass
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        raise NotImplementedError  # implement this method in your engine subclass
+
+    @classmethod
+    def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
+        raise NotImplementedError  # implement this method in your engine subclass
+
+    @classmethod
+    def __download_and_extract_app(cls, remote_url, download_location, timeout=120):
+
+        # Create a temp download directory
+        temp_download_dir = tempfile.mkdtemp()
+        temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
+
+        try:
+            output_dir_name = os.path.basename(remote_url)
+            for fmt in cls.supported_formats:
+                output_dir_name = output_dir_name.split(fmt)[0]
+
+            if os.path.exists(os.path.join(download_location, output_dir_name)):
+                logger.error(f"Engine download for {output_dir_name} already exists")
+                return
+
+            if not os.path.exists(temp_downloaded_file_path):
+                # Make a GET request to the URL with stream=True to enable streaming
+                logger.info(f"Downloading {output_dir_name} from {remote_url}")
+                response = requests.get(remote_url, stream=True, timeout=timeout)
+
+                # Check if the request was successful
+                if response.status_code == 200:
+                    # Get the total file size from the "Content-Length" header
+                    file_size = int(response.headers.get("Content-Length", 0))
+
+                    # Create a progress bar using tqdm
+                    progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
+
+                    # Open a file for writing in binary mode
+                    with open(temp_downloaded_file_path, "wb") as file:
+                        for chunk in response.iter_content(chunk_size=1024):
+                            if chunk:
+                                # Write the chunk to the file
+                                file.write(chunk)
+                                # Update the progress bar
+                                progress_bar.update(len(chunk))
+
+                    # Close the progress bar
+                    progress_bar.close()
+                    logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
+                else:
+                    logger.error(f"Failed to download the file. Status code: {response.status_code}")
+                    return
+
+            os.makedirs(download_location, exist_ok=True)
+
+            # Extract the downloaded file
+            # Process .tar.xz files
+            if temp_downloaded_file_path.lower().endswith('.tar.xz'):
+                try:
+                    with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
+                        tar.extractall(path=download_location)
+
+                    logger.info(
+                        f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
+                except tarfile.TarError as e:
+                    logger.error(f'Error extracting {temp_downloaded_file_path}: {e}')
+                except FileNotFoundError:
+                    logger.error(f'File not found: {temp_downloaded_file_path}')
+
+            # Process .zip files
+            elif temp_downloaded_file_path.lower().endswith('.zip'):
+                try:
+                    with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
+                        zip_ref.extractall(download_location)
+                    logger.info(
+                        f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
+                except zipfile.BadZipFile as e:
+                    logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
+                except FileNotFoundError:
+                    logger.error(f'File not found: {temp_downloaded_file_path}')
+
+            # Process .dmg files (macOS only)
+            elif temp_downloaded_file_path.lower().endswith('.dmg'):
+                import dmglib
+                dmg = dmglib.DiskImage(temp_downloaded_file_path)
+                for mount_point in dmg.attach():
+                    try:
+                        copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
+                        logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
+                    except FileNotFoundError:
+                        logger.error(f'Error: The source .app bundle does not exist.')
+                    except PermissionError:
+                        logger.error(f'Error: Permission denied to copy {download_location}.')
+                    except Exception as e:
+                        logger.error(f'An error occurred: {e}')
+                dmg.detach()
+
+            else:
+                logger.error("Unknown file. Unable to extract binary.")
+
+        except Exception as e:
+            logger.exception(e)
+
+        # remove downloaded file on completion
+        shutil.rmtree(temp_download_dir)
+        return download_location
+
+
+# Function to copy directory contents but ignore symbolic links and hidden files
+def copy_directory_contents(src_dir, dest_dir):
+    try:
+        # Create the destination directory if it doesn't exist
+        os.makedirs(dest_dir, exist_ok=True)
+
+        for item in os.listdir(src_dir):
+            item_path = os.path.join(src_dir, item)
+
+            # Ignore symbolic links
+            if os.path.islink(item_path):
+                continue
+
+            # Ignore hidden files or directories (those starting with a dot)
+            if not item.startswith('.'):
+                dest_item_path = os.path.join(dest_dir, item)
+
+                # If it's a directory, recursively copy its contents
+                if os.path.isdir(item_path):
+                    copy_directory_contents(item_path, dest_item_path)
+                else:
+                    # Otherwise, copy the file
+                    shutil.copy2(item_path, dest_item_path)
+
+    except Exception as e:
+        logger.exception(f"Error copying directory contents: {e}")

src/engines/core/base_engine.py

@@ -21,7 +21,7 @@ class BaseRenderEngine(object):
 
     @classmethod
     def name(cls):
-        return cls.__name__.lower()
+        return str(cls.__name__).lower()
 
     @classmethod
     def default_renderer_path(cls):
@@ -39,6 +39,14 @@ class BaseRenderEngine(object):
     def version(self):
         raise NotImplementedError("version not implemented")
 
+    @staticmethod
+    def downloader():  # override when subclassing if using a downloader class
+        return None
+
+    @staticmethod
+    def worker_class():  # override when subclassing to link worker class
+        raise NotImplementedError("Worker class not implemented")
+
     def get_help(self):
         path = self.renderer_path()
         if not path:

src/engines/core/downloader_core.py

@@ -1,139 +0,0 @@
-import logging
-import os
-import shutil
-import tarfile
-import tempfile
-import zipfile
-
-import requests
-from tqdm import tqdm
-
-supported_formats = ['.zip', '.tar.xz', '.dmg']
-logger = logging.getLogger()
-
-
-def download_and_extract_app(remote_url, download_location, timeout=120):
-
-    # Create a temp download directory
-    temp_download_dir = tempfile.mkdtemp()
-    temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
-
-    try:
-        output_dir_name = os.path.basename(remote_url)
-        for fmt in supported_formats:
-            output_dir_name = output_dir_name.split(fmt)[0]
-
-        if os.path.exists(os.path.join(download_location, output_dir_name)):
-            logger.error(f"Engine download for {output_dir_name} already exists")
-            return
-
-        if not os.path.exists(temp_downloaded_file_path):
-            # Make a GET request to the URL with stream=True to enable streaming
-            logger.info(f"Downloading {output_dir_name} from {remote_url}")
-            response = requests.get(remote_url, stream=True, timeout=timeout)
-
-            # Check if the request was successful
-            if response.status_code == 200:
-                # Get the total file size from the "Content-Length" header
-                file_size = int(response.headers.get("Content-Length", 0))
-
-                # Create a progress bar using tqdm
-                progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
-
-                # Open a file for writing in binary mode
-                with open(temp_downloaded_file_path, "wb") as file:
-                    for chunk in response.iter_content(chunk_size=1024):
-                        if chunk:
-                            # Write the chunk to the file
-                            file.write(chunk)
-                            # Update the progress bar
-                            progress_bar.update(len(chunk))
-
-                # Close the progress bar
-                progress_bar.close()
-                logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
-            else:
-                logger.error(f"Failed to download the file. Status code: {response.status_code}")
-                return
-
-        os.makedirs(download_location, exist_ok=True)
-
-        # Extract the downloaded file
-        # Process .tar.xz files
-        if temp_downloaded_file_path.lower().endswith('.tar.xz'):
-            try:
-                with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
-                    tar.extractall(path=download_location)
-
-                logger.info(
-                    f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
-            except tarfile.TarError as e:
-                logger.error(f'Error extracting {temp_downloaded_file_path}: {e}')
-            except FileNotFoundError:
-                logger.error(f'File not found: {temp_downloaded_file_path}')
-
-        # Process .zip files
-        elif temp_downloaded_file_path.lower().endswith('.zip'):
-            try:
-                with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
-                    zip_ref.extractall(download_location)
-                logger.info(
-                    f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
-            except zipfile.BadZipFile as e:
-                logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
-            except FileNotFoundError:
-                logger.error(f'File not found: {temp_downloaded_file_path}')
-
-        # Process .dmg files (macOS only)
-        elif temp_downloaded_file_path.lower().endswith('.dmg'):
-            import dmglib
-            dmg = dmglib.DiskImage(temp_downloaded_file_path)
-            for mount_point in dmg.attach():
-                try:
-                    copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
-                    logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
-                except FileNotFoundError:
-                    logger.error(f'Error: The source .app bundle does not exist.')
-                except PermissionError:
-                    logger.error(f'Error: Permission denied to copy {download_location}.')
-                except Exception as e:
-                    logger.error(f'An error occurred: {e}')
-            dmg.detach()
-
-        else:
-            logger.error("Unknown file. Unable to extract binary.")
-
-    except Exception as e:
-        logger.exception(e)
-
-    # remove downloaded file on completion
-    shutil.rmtree(temp_download_dir)
-    return download_location
-
-
-# Function to copy directory contents but ignore symbolic links and hidden files
-def copy_directory_contents(src_dir, dest_dir):
-    try:
-        # Create the destination directory if it doesn't exist
-        os.makedirs(dest_dir, exist_ok=True)
-
-        for item in os.listdir(src_dir):
-            item_path = os.path.join(src_dir, item)
-
-            # Ignore symbolic links
-            if os.path.islink(item_path):
-                continue
-
-            # Ignore hidden files or directories (those starting with a dot)
-            if not item.startswith('.'):
-                dest_item_path = os.path.join(dest_dir, item)
-
-                # If it's a directory, recursively copy its contents
-                if os.path.isdir(item_path):
-                    copy_directory_contents(item_path, dest_item_path)
-                else:
-                    # Otherwise, copy the file
-                    shutil.copy2(item_path, dest_item_path)
-
-    except Exception as e:
-        logger.exception(f"Error copying directory contents: {e}")

src/engines/core/worker_factory.py

@@ -1,61 +0,0 @@
-import logging
-
-from src.engines.engine_manager import EngineManager
-
-logger = logging.getLogger()
-
-
-class RenderWorkerFactory:
-
-    @staticmethod
-    def supported_classes():
-        # to add support for any additional RenderWorker classes, import their classes and add to list here
-        from src.engines.blender.blender_worker import BlenderRenderWorker
-        from src.engines.aerender.aerender_worker import AERenderWorker
-        from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker
-        classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker]
-        return classes
-
-    @staticmethod
-    def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
-
-        worker_class = RenderWorkerFactory.class_for_name(renderer)
-
-        # check to make sure we have versions installed
-        all_versions = EngineManager.all_versions_for_engine(renderer)
-        if not all_versions:
-            raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
-
-        # Find the path to the requested engine version or use default
-        engine_path = None if engine_version else all_versions[0]['path']
-        if engine_version:
-            for ver in all_versions:
-                if ver['version'] == engine_version:
-                    engine_path = ver['path']
-                    break
-
-            # Download the required engine if not found locally
-            if not engine_path:
-                download_result = EngineManager.download_engine(renderer, engine_version)
-                if not download_result:
-                    raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
-                engine_path = download_result['path']
-                logger.info("Engine downloaded. Creating worker.")
-
-        if not engine_path:
-            raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
-
-        return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
-                            parent=parent, name=name)
-
-    @staticmethod
-    def supported_renderers():
-        return [x.engine.name() for x in RenderWorkerFactory.supported_classes()]
-
-    @staticmethod
-    def class_for_name(name):
-        name = name.lower()
-        for render_class in RenderWorkerFactory.supported_classes():
-            if render_class.engine.name() == name:
-                return render_class
-        raise LookupError(f'Cannot find class for name: {name}')

src/engines/engine_manager.py

@@ -3,9 +3,7 @@ import os
 import shutil
 import threading
 
-from src.engines.blender.blender_downloader import BlenderDownloader
 from src.engines.blender.blender_engine import Blender
-from src.engines.ffmpeg.ffmpeg_downloader import FFMPEGDownloader
 from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
 from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu
 
@@ -14,21 +12,26 @@ logger = logging.getLogger()
 
 class EngineManager:
 
-    engines_path = "~/zordon-uploads/engines"
+    engines_path = None
-    downloader_classes = {
-        "blender": BlenderDownloader,
-        "ffmpeg": FFMPEGDownloader,
-        # Add more engine types and corresponding downloader classes as needed
-    }
 
-    @classmethod
+    @staticmethod
-    def supported_engines(cls):
+    def supported_engines():
         return [Blender, FFMPEG]
 
+    @classmethod
+    def engine_with_name(cls, engine_name):
+        for obj in cls.supported_engines():
+            if obj.name().lower() == engine_name.lower():
+                return obj
+
     @classmethod
     def all_engines(cls):
-        results = []
+
+        if not cls.engines_path:
+            raise FileNotFoundError("Engines path must be set before requesting downloads")
+
         # Parse downloaded engine directory
+        results = []
         try:
             all_items = os.listdir(cls.engines_path)
             all_directories = [item for item in all_items if os.path.isdir(os.path.join(cls.engines_path, item))]
@@ -57,8 +60,8 @@ class EngineManager:
 
                 result_dict['path'] = path
                 results.append(result_dict)
-        except FileNotFoundError:
+        except FileNotFoundError as e:
-            logger.warning("Cannot find local engines download directory")
+            logger.warning(f"Cannot find local engines download directory: {e}")
 
         # add system installs to this list
         for eng in cls.supported_engines():
@@ -99,15 +102,16 @@ class EngineManager:
     @classmethod
     def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None):
         try:
-            return cls.downloader_classes[engine].version_is_available_to_download(version=version, system_os=system_os,
+            downloader = cls.engine_with_name(engine).downloader()
-                                                                                   cpu=cpu)
+            return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu)
         except Exception as e:
             return None
 
     @classmethod
     def find_most_recent_version(cls, engine=None, system_os=None, cpu=None, lts_only=False):
         try:
-            return cls.downloader_classes[engine].find_most_recent_version(system_os=system_os, cpu=cpu)
+            downloader = cls.engine_with_name(engine).downloader()
+            return downloader.find_most_recent_version(system_os=system_os, cpu=cpu)
         except Exception as e:
             return None
 
@@ -119,13 +123,19 @@ class EngineManager:
             return existing_download
 
         # Check if the provided engine type is valid
-        if engine not in cls.downloader_classes:
+        engine_to_download = cls.engine_with_name(engine)
+        if not engine_to_download:
             logger.error("No valid engine found")
             return
+        elif not engine_to_download.downloader():
+            logger.warning("No valid downloader for this engine. Please update this software manually.")
+            return
+        elif not cls.engines_path:
+            raise FileNotFoundError("Engines path must be set before requesting downloads")
 
         # Get the appropriate downloader class based on the engine type
-        cls.downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
+        engine_to_download.downloader().download_engine(version, download_location=cls.engines_path,
-                                                       system_os=system_os, cpu=cpu, timeout=300)
+                                                                  system_os=system_os, cpu=cpu, timeout=300)
 
         # Check that engine was properly downloaded
         found_engine = cls.is_version_downloaded(engine, version, system_os, cpu)
@@ -148,30 +158,66 @@ class EngineManager:
 
     @classmethod
     def update_all_engines(cls):
-        def engine_update_task(engine, engine_downloader):
+        def engine_update_task(engine):
-            logger.debug(f"Checking for updates to {engine}")
+            logger.debug(f"Checking for updates to {engine.name()}")
-            latest_version = engine_downloader.find_most_recent_version()
+            latest_version = engine.downloader().find_most_recent_version()
             if latest_version:
-                logger.debug(f"Latest version of {engine} available: {latest_version.get('version')}")
+                logger.debug(f"Latest version of {engine.name()} available: {latest_version.get('version')}")
                 if not cls.is_version_downloaded(engine, latest_version.get('version')):
-                    logger.info(f"Downloading {engine} ({latest_version['version']})")
+                    logger.info(f"Downloading {engine.name()} ({latest_version['version']})")
-                    cls.download_engine(engine=engine, version=latest_version['version'])
+                    cls.download_engine(engine=engine.name(), version=latest_version['version'])
             else:
-                logger.warning(f"Unable to get latest version for {engine}")
+                logger.warning(f"Unable to get check for updates for {engine.name()}")
 
         logger.info(f"Checking for updates for render engines...")
         threads = []
-        for engine, engine_downloader in cls.downloader_classes.items():
+        for engine in cls.supported_engines():
-            thread = threading.Thread(target=engine_update_task, args=(engine, engine_downloader))
+            if engine.downloader():
-            threads.append(thread)
+                thread = threading.Thread(target=engine_update_task, args=(engine,))
-            thread.start()
+                threads.append(thread)
+                thread.start()
 
         for thread in threads:  # wait to finish
             thread.join()
 
+    @classmethod
+    def create_worker(cls, renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
+
+        worker_class = cls.engine_with_name(renderer).worker_class()
+
+        # check to make sure we have versions installed
+        all_versions = EngineManager.all_versions_for_engine(renderer)
+        if not all_versions:
+            raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
+
+        # Find the path to the requested engine version or use default
+        engine_path = None if engine_version else all_versions[0]['path']
+        if engine_version:
+            for ver in all_versions:
+                if ver['version'] == engine_version:
+                    engine_path = ver['path']
+                    break
+
+            # Download the required engine if not found locally
+            if not engine_path:
+                download_result = EngineManager.download_engine(renderer, engine_version)
+                if not download_result:
+                    raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
+                engine_path = download_result['path']
+                logger.info("Engine downloaded. Creating worker.")
+
+        if not engine_path:
+            raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
+
+        return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
+                            parent=parent, name=name)
+
+
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
     # print(EngineManager.newest_engine_version('blender', 'macos', 'arm64'))
-    EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a')
+    # EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a')
-
+    EngineManager.engines_path = "/Users/brettwilliams/zordon-uploads/engines"
+    # print(EngineManager.is_version_downloaded("ffmpeg", "6.0"))
+    print(EngineManager.all_engines())

src/engines/ffmpeg/ffmpeg_downloader.py

@@ -4,14 +4,16 @@ import re
 
 import requests
 
-from src.engines.core.downloader_core import download_and_extract_app
+from src.engines.core.base_downloader import EngineDownloader
+from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
 from src.utilities.misc_helper import current_system_cpu, current_system_os
 
 logger = logging.getLogger()
-supported_formats = ['.zip', '.tar.xz', '.dmg']
 
 
-class FFMPEGDownloader:
+class FFMPEGDownloader(EngineDownloader):
+
+    engine = FFMPEG
 
     # macOS FFMPEG mirror maintained by Evermeet - https://evermeet.cx/ffmpeg/
     macos_url = "https://evermeet.cx/pub/ffmpeg/"
@@ -88,17 +90,7 @@ class FFMPEGDownloader:
         return releases
 
     @classmethod
-    def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
+    def __all_versions(cls, system_os=None, cpu=None):
-        try:
-            system_os = system_os or current_system_os()
-            cpu = cpu or current_system_cpu()
-            return cls.all_versions(system_os, cpu)[0]
-        except TypeError:
-            pass
-        return None
-
-    @classmethod
-    def all_versions(cls, system_os=None, cpu=None):
         system_os = system_os or current_system_os()
         cpu = cpu or current_system_cpu()
         versions_per_os = {'linux': cls.__get_linux_versions, 'macos': cls.__get_macos_versions,
@@ -115,13 +107,6 @@ class FFMPEGDownloader:
                             'version': version})
         return results
 
-    @classmethod
-    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
-        for ver in cls.all_versions(system_os, cpu):
-            if ver['version'] == version:
-                return ver
-        return None
-
     @classmethod
     def __get_remote_url_for_version(cls, version, system_os, cpu):
         # Platform specific naming cleanup
@@ -141,13 +126,30 @@ class FFMPEGDownloader:
             logger.error("Unknown system os")
         return remote_url
 
+    @classmethod
+    def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
+        try:
+            system_os = system_os or current_system_os()
+            cpu = cpu or current_system_cpu()
+            return cls.__all_versions(system_os, cpu)[0]
+        except (IndexError, requests.exceptions.RequestException):
+            logger.error(f"Cannot get most recent version of ffmpeg")
+        return {}
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        for ver in cls.__all_versions(system_os, cpu):
+            if ver['version'] == version:
+                return ver
+        return None
+
     @classmethod
     def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
         system_os = system_os or current_system_os()
         cpu = cpu or current_system_cpu()
 
         # Verify requested version is available
-        found_version = [item for item in cls.all_versions(system_os, cpu) if item['version'] == version]
+        found_version = [item for item in cls.__all_versions(system_os, cpu) if item['version'] == version]
         if not found_version:
             logger.error(f"Cannot find FFMPEG version {version} for {system_os} and {cpu}")
             return
@@ -160,7 +162,7 @@ class FFMPEGDownloader:
         # Download and extract
         try:
             logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}")
-            download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout)
+            cls.__download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout)
 
             # naming cleanup to match existing naming convention
             output_path = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')

src/engines/ffmpeg/ffmpeg_engine.py

@@ -7,6 +7,16 @@ class FFMPEG(BaseRenderEngine):
 
     binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
 
+    @staticmethod
+    def downloader():
+        from src.engines.ffmpeg.ffmpeg_downloader import FFMPEGDownloader
+        return FFMPEGDownloader
+
+    @staticmethod
+    def worker_class():
+        from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker
+        return FFMPEGRenderWorker
+
     def version(self):
         version = None
         try: