From dcc0504d3c2592dea1ab2eb7938edce9b9da2730 Mon Sep 17 00:00:00 2001 From: Brett Date: Sun, 29 Oct 2023 20:57:26 -0500 Subject: [PATCH] Engine and downloader refactoring (#50) * Make downloaders subclass of base_downloader.py * Link engines and downloaders together for all engines * Replace / merge worker_factory.py with engine_manager.py --- src/api/add_job_helpers.py | 12 +- src/api/api_server.py | 11 +- src/engines/blender/blender_downloader.py | 50 +++---- src/engines/blender/blender_engine.py | 10 ++ src/engines/core/base_downloader.py | 158 ++++++++++++++++++++++ src/engines/core/base_engine.py | 10 +- src/engines/core/downloader_core.py | 139 ------------------- src/engines/core/worker_factory.py | 61 --------- src/engines/engine_manager.py | 110 ++++++++++----- src/engines/ffmpeg/ffmpeg_downloader.py | 48 +++---- src/engines/ffmpeg/ffmpeg_engine.py | 10 ++ 11 files changed, 328 insertions(+), 291 deletions(-) create mode 100644 src/engines/core/base_downloader.py delete mode 100644 src/engines/core/downloader_core.py delete mode 100644 src/engines/core/worker_factory.py diff --git a/src/api/add_job_helpers.py b/src/api/add_job_helpers.py index 28280b3..3400336 100644 --- a/src/api/add_job_helpers.py +++ b/src/api/add_job_helpers.py @@ -11,7 +11,7 @@ from tqdm import tqdm from werkzeug.utils import secure_filename from src.distributed_job_manager import DistributedJobManager -from src.engines.core.worker_factory import RenderWorkerFactory +from src.engines.engine_manager import EngineManager from src.render_queue import RenderQueue logger = logging.getLogger() @@ -149,11 +149,11 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir, enable_spl logger.debug(f"New job output path: {output_path}") # create & configure jobs - worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'], - input_path=loaded_project_local_path, - output_path=output_path, - engine_version=job_data.get('engine_version'), - args=job_data.get('args', {})) + worker = EngineManager.create_worker(renderer=job_data['renderer'], + input_path=loaded_project_local_path, + output_path=output_path, + engine_version=job_data.get('engine_version'), + args=job_data.get('args', {})) worker.status = job_data.get("initial_status", worker.status) worker.parent = job_data.get("parent", worker.parent) worker.name = job_data.get("name", worker.name) diff --git a/src/api/api_server.py b/src/api/api_server.py index 396c9d8..5502286 100755 --- a/src/api/api_server.py +++ b/src/api/api_server.py @@ -21,7 +21,6 @@ from src.api.server_proxy import RenderServerProxy from src.api.add_job_helpers import handle_uploaded_project_files, process_zipped_project, create_render_jobs from src.distributed_job_manager import DistributedJobManager from src.engines.core.base_worker import string_to_status, RenderStatus -from src.engines.core.worker_factory import RenderWorkerFactory from src.engines.engine_manager import EngineManager from src.render_queue import RenderQueue, JobNotFoundError from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, current_system_os_version @@ -390,7 +389,7 @@ def clear_history(): @server.route('/api/status') def status(): renderer_data = {} - for render_class in RenderWorkerFactory.supported_classes(): + for render_class in EngineManager.supported_engines(): if EngineManager.all_versions_for_engine(render_class.name): # only return renderers installed on host renderer_data[render_class.engine.name()] = \ {'versions': EngineManager.all_versions_for_engine(render_class.engine.name()), @@ -418,8 +417,8 @@ def status(): @server.get('/api/renderer_info') def renderer_info(): renderer_data = {} - for engine_name in RenderWorkerFactory.supported_renderers(): - engine = RenderWorkerFactory.class_for_name(engine_name).engine + for engine_name in EngineManager.supported_engines(): + engine = EngineManager.engine_with_name(engine_name) # Get all installed versions of engine installed_versions = EngineManager.all_versions_for_engine(engine_name) @@ -482,7 +481,7 @@ def delete_engine_download(): @server.get('/api/renderer//args') def get_renderer_args(renderer): try: - renderer_engine_class = RenderWorkerFactory.class_for_name(renderer).engine() + renderer_engine_class = EngineManager.engine_with_name(renderer) return renderer_engine_class.get_arguments() except LookupError: return f"Cannot find renderer '{renderer}'", 400 @@ -490,7 +489,7 @@ def get_renderer_args(renderer): @server.route('/upload') def upload_file_page(): - return render_template('upload.html', supported_renderers=RenderWorkerFactory.supported_renderers()) + return render_template('upload.html', supported_renderers=EngineManager.supported_engines()) def start_server(background_thread=False): diff --git a/src/engines/blender/blender_downloader.py b/src/engines/blender/blender_downloader.py index 060f59d..210cd29 100644 --- a/src/engines/blender/blender_downloader.py +++ b/src/engines/blender/blender_downloader.py @@ -3,7 +3,8 @@ import re import requests -from src.engines.core.downloader_core import download_and_extract_app +from src.engines.blender.blender_engine import Blender +from src.engines.core.base_downloader import EngineDownloader from src.utilities.misc_helper import current_system_os, current_system_cpu # url = "https://download.blender.org/release/" @@ -13,10 +14,12 @@ logger = logging.getLogger() supported_formats = ['.zip', '.tar.xz', '.dmg'] -class BlenderDownloader: +class BlenderDownloader(EngineDownloader): + + engine = Blender @staticmethod - def get_major_versions(): + def __get_major_versions(): try: response = requests.get(url, timeout=5) response.raise_for_status() @@ -30,10 +33,10 @@ class BlenderDownloader: return major_versions except requests.exceptions.RequestException as e: logger.error(f"Error: {e}") - return None + return [] @staticmethod - def get_minor_versions(major_version, system_os=None, cpu=None): + def __get_minor_versions(major_version, system_os=None, cpu=None): try: base_url = url + 'Blender' + major_version @@ -63,17 +66,8 @@ class BlenderDownloader: logger.exception(e) return [] - @classmethod - def version_is_available_to_download(cls, version, system_os=None, cpu=None): - requested_major_version = '.'.join(version.split('.')[:2]) - minor_versions = cls.get_minor_versions(requested_major_version, system_os, cpu) - for minor in minor_versions: - if minor['version'] == version: - return minor - return None - @staticmethod - def find_LTS_versions(): + def __find_LTS_versions(): response = requests.get('https://www.blender.org/download/lts/') response.raise_for_status() @@ -87,11 +81,21 @@ class BlenderDownloader: @classmethod def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False): try: - major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0] - most_recent = cls.get_minor_versions(major_version=major_version, system_os=system_os, cpu=cpu) + major_version = cls.__find_LTS_versions()[0] if lts_only else cls.__get_major_versions()[0] + most_recent = cls.__get_minor_versions(major_version=major_version, system_os=system_os, cpu=cpu) return most_recent[0] - except IndexError: - logger.error("Cannot find a most recent version") + except (IndexError, requests.exceptions.RequestException): + logger.error(f"Cannot get most recent version of blender") + return {} + + @classmethod + def version_is_available_to_download(cls, version, system_os=None, cpu=None): + requested_major_version = '.'.join(version.split('.')[:2]) + minor_versions = cls.__get_minor_versions(requested_major_version, system_os, cpu) + for minor in minor_versions: + if minor['version'] == version: + return minor + return None @classmethod def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120): @@ -101,11 +105,11 @@ class BlenderDownloader: try: logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}") major_version = '.'.join(version.split('.')[:2]) - minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version] + minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if x['version'] == version] # we get the URL instead of calculating it ourselves. May change this - download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location, - timeout=timeout) + cls.__download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location, + timeout=timeout) except IndexError: logger.error("Cannot find requested engine") @@ -113,5 +117,5 @@ class BlenderDownloader: if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - print(BlenderDownloader.get_major_versions()) + print(BlenderDownloader.__get_major_versions()) diff --git a/src/engines/blender/blender_engine.py b/src/engines/blender/blender_engine.py index 7ce3106..3066867 100644 --- a/src/engines/blender/blender_engine.py +++ b/src/engines/blender/blender_engine.py @@ -13,6 +13,16 @@ class Blender(BaseRenderEngine): supported_extensions = ['.blend'] binary_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender'} + @staticmethod + def downloader(): + from src.engines.blender.blender_downloader import BlenderDownloader + return BlenderDownloader + + @staticmethod + def worker_class(): + from src.engines.blender.blender_worker import BlenderRenderWorker + return BlenderRenderWorker + def version(self): version = None try: diff --git a/src/engines/core/base_downloader.py b/src/engines/core/base_downloader.py new file mode 100644 index 0000000..1d925b7 --- /dev/null +++ b/src/engines/core/base_downloader.py @@ -0,0 +1,158 @@ +import logging +import os +import shutil +import tarfile +import tempfile +import zipfile + +import requests +from tqdm import tqdm + +logger = logging.getLogger() + + +class EngineDownloader: + + supported_formats = ['.zip', '.tar.xz', '.dmg'] + + def __init__(self): + pass + + @classmethod + def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False): + raise NotImplementedError # implement this method in your engine subclass + + @classmethod + def version_is_available_to_download(cls, version, system_os=None, cpu=None): + raise NotImplementedError # implement this method in your engine subclass + + @classmethod + def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120): + raise NotImplementedError # implement this method in your engine subclass + + @classmethod + def __download_and_extract_app(cls, remote_url, download_location, timeout=120): + + # Create a temp download directory + temp_download_dir = tempfile.mkdtemp() + temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url)) + + try: + output_dir_name = os.path.basename(remote_url) + for fmt in cls.supported_formats: + output_dir_name = output_dir_name.split(fmt)[0] + + if os.path.exists(os.path.join(download_location, output_dir_name)): + logger.error(f"Engine download for {output_dir_name} already exists") + return + + if not os.path.exists(temp_downloaded_file_path): + # Make a GET request to the URL with stream=True to enable streaming + logger.info(f"Downloading {output_dir_name} from {remote_url}") + response = requests.get(remote_url, stream=True, timeout=timeout) + + # Check if the request was successful + if response.status_code == 200: + # Get the total file size from the "Content-Length" header + file_size = int(response.headers.get("Content-Length", 0)) + + # Create a progress bar using tqdm + progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) + + # Open a file for writing in binary mode + with open(temp_downloaded_file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # Write the chunk to the file + file.write(chunk) + # Update the progress bar + progress_bar.update(len(chunk)) + + # Close the progress bar + progress_bar.close() + logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}") + else: + logger.error(f"Failed to download the file. Status code: {response.status_code}") + return + + os.makedirs(download_location, exist_ok=True) + + # Extract the downloaded file + # Process .tar.xz files + if temp_downloaded_file_path.lower().endswith('.tar.xz'): + try: + with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar: + tar.extractall(path=download_location) + + logger.info( + f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except tarfile.TarError as e: + logger.error(f'Error extracting {temp_downloaded_file_path}: {e}') + except FileNotFoundError: + logger.error(f'File not found: {temp_downloaded_file_path}') + + # Process .zip files + elif temp_downloaded_file_path.lower().endswith('.zip'): + try: + with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref: + zip_ref.extractall(download_location) + logger.info( + f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except zipfile.BadZipFile as e: + logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.') + except FileNotFoundError: + logger.error(f'File not found: {temp_downloaded_file_path}') + + # Process .dmg files (macOS only) + elif temp_downloaded_file_path.lower().endswith('.dmg'): + import dmglib + dmg = dmglib.DiskImage(temp_downloaded_file_path) + for mount_point in dmg.attach(): + try: + copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name)) + logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except FileNotFoundError: + logger.error(f'Error: The source .app bundle does not exist.') + except PermissionError: + logger.error(f'Error: Permission denied to copy {download_location}.') + except Exception as e: + logger.error(f'An error occurred: {e}') + dmg.detach() + + else: + logger.error("Unknown file. Unable to extract binary.") + + except Exception as e: + logger.exception(e) + + # remove downloaded file on completion + shutil.rmtree(temp_download_dir) + return download_location + + +# Function to copy directory contents but ignore symbolic links and hidden files +def copy_directory_contents(src_dir, dest_dir): + try: + # Create the destination directory if it doesn't exist + os.makedirs(dest_dir, exist_ok=True) + + for item in os.listdir(src_dir): + item_path = os.path.join(src_dir, item) + + # Ignore symbolic links + if os.path.islink(item_path): + continue + + # Ignore hidden files or directories (those starting with a dot) + if not item.startswith('.'): + dest_item_path = os.path.join(dest_dir, item) + + # If it's a directory, recursively copy its contents + if os.path.isdir(item_path): + copy_directory_contents(item_path, dest_item_path) + else: + # Otherwise, copy the file + shutil.copy2(item_path, dest_item_path) + + except Exception as e: + logger.exception(f"Error copying directory contents: {e}") diff --git a/src/engines/core/base_engine.py b/src/engines/core/base_engine.py index ccb3b07..c09d56b 100644 --- a/src/engines/core/base_engine.py +++ b/src/engines/core/base_engine.py @@ -21,7 +21,7 @@ class BaseRenderEngine(object): @classmethod def name(cls): - return cls.__name__.lower() + return str(cls.__name__).lower() @classmethod def default_renderer_path(cls): @@ -39,6 +39,14 @@ class BaseRenderEngine(object): def version(self): raise NotImplementedError("version not implemented") + @staticmethod + def downloader(): # override when subclassing if using a downloader class + return None + + @staticmethod + def worker_class(): # override when subclassing to link worker class + raise NotImplementedError("Worker class not implemented") + def get_help(self): path = self.renderer_path() if not path: diff --git a/src/engines/core/downloader_core.py b/src/engines/core/downloader_core.py deleted file mode 100644 index c0cb6e4..0000000 --- a/src/engines/core/downloader_core.py +++ /dev/null @@ -1,139 +0,0 @@ -import logging -import os -import shutil -import tarfile -import tempfile -import zipfile - -import requests -from tqdm import tqdm - -supported_formats = ['.zip', '.tar.xz', '.dmg'] -logger = logging.getLogger() - - -def download_and_extract_app(remote_url, download_location, timeout=120): - - # Create a temp download directory - temp_download_dir = tempfile.mkdtemp() - temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url)) - - try: - output_dir_name = os.path.basename(remote_url) - for fmt in supported_formats: - output_dir_name = output_dir_name.split(fmt)[0] - - if os.path.exists(os.path.join(download_location, output_dir_name)): - logger.error(f"Engine download for {output_dir_name} already exists") - return - - if not os.path.exists(temp_downloaded_file_path): - # Make a GET request to the URL with stream=True to enable streaming - logger.info(f"Downloading {output_dir_name} from {remote_url}") - response = requests.get(remote_url, stream=True, timeout=timeout) - - # Check if the request was successful - if response.status_code == 200: - # Get the total file size from the "Content-Length" header - file_size = int(response.headers.get("Content-Length", 0)) - - # Create a progress bar using tqdm - progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) - - # Open a file for writing in binary mode - with open(temp_downloaded_file_path, "wb") as file: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - # Write the chunk to the file - file.write(chunk) - # Update the progress bar - progress_bar.update(len(chunk)) - - # Close the progress bar - progress_bar.close() - logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}") - else: - logger.error(f"Failed to download the file. Status code: {response.status_code}") - return - - os.makedirs(download_location, exist_ok=True) - - # Extract the downloaded file - # Process .tar.xz files - if temp_downloaded_file_path.lower().endswith('.tar.xz'): - try: - with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar: - tar.extractall(path=download_location) - - logger.info( - f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') - except tarfile.TarError as e: - logger.error(f'Error extracting {temp_downloaded_file_path}: {e}') - except FileNotFoundError: - logger.error(f'File not found: {temp_downloaded_file_path}') - - # Process .zip files - elif temp_downloaded_file_path.lower().endswith('.zip'): - try: - with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref: - zip_ref.extractall(download_location) - logger.info( - f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') - except zipfile.BadZipFile as e: - logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.') - except FileNotFoundError: - logger.error(f'File not found: {temp_downloaded_file_path}') - - # Process .dmg files (macOS only) - elif temp_downloaded_file_path.lower().endswith('.dmg'): - import dmglib - dmg = dmglib.DiskImage(temp_downloaded_file_path) - for mount_point in dmg.attach(): - try: - copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name)) - logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}') - except FileNotFoundError: - logger.error(f'Error: The source .app bundle does not exist.') - except PermissionError: - logger.error(f'Error: Permission denied to copy {download_location}.') - except Exception as e: - logger.error(f'An error occurred: {e}') - dmg.detach() - - else: - logger.error("Unknown file. Unable to extract binary.") - - except Exception as e: - logger.exception(e) - - # remove downloaded file on completion - shutil.rmtree(temp_download_dir) - return download_location - - -# Function to copy directory contents but ignore symbolic links and hidden files -def copy_directory_contents(src_dir, dest_dir): - try: - # Create the destination directory if it doesn't exist - os.makedirs(dest_dir, exist_ok=True) - - for item in os.listdir(src_dir): - item_path = os.path.join(src_dir, item) - - # Ignore symbolic links - if os.path.islink(item_path): - continue - - # Ignore hidden files or directories (those starting with a dot) - if not item.startswith('.'): - dest_item_path = os.path.join(dest_dir, item) - - # If it's a directory, recursively copy its contents - if os.path.isdir(item_path): - copy_directory_contents(item_path, dest_item_path) - else: - # Otherwise, copy the file - shutil.copy2(item_path, dest_item_path) - - except Exception as e: - logger.exception(f"Error copying directory contents: {e}") diff --git a/src/engines/core/worker_factory.py b/src/engines/core/worker_factory.py deleted file mode 100644 index 1c3ac2d..0000000 --- a/src/engines/core/worker_factory.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging - -from src.engines.engine_manager import EngineManager - -logger = logging.getLogger() - - -class RenderWorkerFactory: - - @staticmethod - def supported_classes(): - # to add support for any additional RenderWorker classes, import their classes and add to list here - from src.engines.blender.blender_worker import BlenderRenderWorker - from src.engines.aerender.aerender_worker import AERenderWorker - from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker - classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker] - return classes - - @staticmethod - def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None): - - worker_class = RenderWorkerFactory.class_for_name(renderer) - - # check to make sure we have versions installed - all_versions = EngineManager.all_versions_for_engine(renderer) - if not all_versions: - raise FileNotFoundError(f"Cannot find any installed {renderer} engines") - - # Find the path to the requested engine version or use default - engine_path = None if engine_version else all_versions[0]['path'] - if engine_version: - for ver in all_versions: - if ver['version'] == engine_version: - engine_path = ver['path'] - break - - # Download the required engine if not found locally - if not engine_path: - download_result = EngineManager.download_engine(renderer, engine_version) - if not download_result: - raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}") - engine_path = download_result['path'] - logger.info("Engine downloaded. Creating worker.") - - if not engine_path: - raise FileNotFoundError(f"Cannot find requested engine version {engine_version}") - - return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args, - parent=parent, name=name) - - @staticmethod - def supported_renderers(): - return [x.engine.name() for x in RenderWorkerFactory.supported_classes()] - - @staticmethod - def class_for_name(name): - name = name.lower() - for render_class in RenderWorkerFactory.supported_classes(): - if render_class.engine.name() == name: - return render_class - raise LookupError(f'Cannot find class for name: {name}') \ No newline at end of file diff --git a/src/engines/engine_manager.py b/src/engines/engine_manager.py index 0ceda69..83ec0f3 100644 --- a/src/engines/engine_manager.py +++ b/src/engines/engine_manager.py @@ -3,9 +3,7 @@ import os import shutil import threading -from src.engines.blender.blender_downloader import BlenderDownloader from src.engines.blender.blender_engine import Blender -from src.engines.ffmpeg.ffmpeg_downloader import FFMPEGDownloader from src.engines.ffmpeg.ffmpeg_engine import FFMPEG from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu @@ -14,21 +12,26 @@ logger = logging.getLogger() class EngineManager: - engines_path = "~/zordon-uploads/engines" - downloader_classes = { - "blender": BlenderDownloader, - "ffmpeg": FFMPEGDownloader, - # Add more engine types and corresponding downloader classes as needed - } + engines_path = None - @classmethod - def supported_engines(cls): + @staticmethod + def supported_engines(): return [Blender, FFMPEG] + @classmethod + def engine_with_name(cls, engine_name): + for obj in cls.supported_engines(): + if obj.name().lower() == engine_name.lower(): + return obj + @classmethod def all_engines(cls): - results = [] + + if not cls.engines_path: + raise FileNotFoundError("Engines path must be set before requesting downloads") + # Parse downloaded engine directory + results = [] try: all_items = os.listdir(cls.engines_path) all_directories = [item for item in all_items if os.path.isdir(os.path.join(cls.engines_path, item))] @@ -57,8 +60,8 @@ class EngineManager: result_dict['path'] = path results.append(result_dict) - except FileNotFoundError: - logger.warning("Cannot find local engines download directory") + except FileNotFoundError as e: + logger.warning(f"Cannot find local engines download directory: {e}") # add system installs to this list for eng in cls.supported_engines(): @@ -99,15 +102,16 @@ class EngineManager: @classmethod def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None): try: - return cls.downloader_classes[engine].version_is_available_to_download(version=version, system_os=system_os, - cpu=cpu) + downloader = cls.engine_with_name(engine).downloader() + return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu) except Exception as e: return None @classmethod def find_most_recent_version(cls, engine=None, system_os=None, cpu=None, lts_only=False): try: - return cls.downloader_classes[engine].find_most_recent_version(system_os=system_os, cpu=cpu) + downloader = cls.engine_with_name(engine).downloader() + return downloader.find_most_recent_version(system_os=system_os, cpu=cpu) except Exception as e: return None @@ -119,13 +123,19 @@ class EngineManager: return existing_download # Check if the provided engine type is valid - if engine not in cls.downloader_classes: + engine_to_download = cls.engine_with_name(engine) + if not engine_to_download: logger.error("No valid engine found") return + elif not engine_to_download.downloader(): + logger.warning("No valid downloader for this engine. Please update this software manually.") + return + elif not cls.engines_path: + raise FileNotFoundError("Engines path must be set before requesting downloads") # Get the appropriate downloader class based on the engine type - cls.downloader_classes[engine].download_engine(version, download_location=cls.engines_path, - system_os=system_os, cpu=cpu, timeout=300) + engine_to_download.downloader().download_engine(version, download_location=cls.engines_path, + system_os=system_os, cpu=cpu, timeout=300) # Check that engine was properly downloaded found_engine = cls.is_version_downloaded(engine, version, system_os, cpu) @@ -148,30 +158,66 @@ class EngineManager: @classmethod def update_all_engines(cls): - def engine_update_task(engine, engine_downloader): - logger.debug(f"Checking for updates to {engine}") - latest_version = engine_downloader.find_most_recent_version() + def engine_update_task(engine): + logger.debug(f"Checking for updates to {engine.name()}") + latest_version = engine.downloader().find_most_recent_version() if latest_version: - logger.debug(f"Latest version of {engine} available: {latest_version.get('version')}") + logger.debug(f"Latest version of {engine.name()} available: {latest_version.get('version')}") if not cls.is_version_downloaded(engine, latest_version.get('version')): - logger.info(f"Downloading {engine} ({latest_version['version']})") - cls.download_engine(engine=engine, version=latest_version['version']) + logger.info(f"Downloading {engine.name()} ({latest_version['version']})") + cls.download_engine(engine=engine.name(), version=latest_version['version']) else: - logger.warning(f"Unable to get latest version for {engine}") + logger.warning(f"Unable to get check for updates for {engine.name()}") logger.info(f"Checking for updates for render engines...") threads = [] - for engine, engine_downloader in cls.downloader_classes.items(): - thread = threading.Thread(target=engine_update_task, args=(engine, engine_downloader)) - threads.append(thread) - thread.start() + for engine in cls.supported_engines(): + if engine.downloader(): + thread = threading.Thread(target=engine_update_task, args=(engine,)) + threads.append(thread) + thread.start() for thread in threads: # wait to finish thread.join() + @classmethod + def create_worker(cls, renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None): + + worker_class = cls.engine_with_name(renderer).worker_class() + + # check to make sure we have versions installed + all_versions = EngineManager.all_versions_for_engine(renderer) + if not all_versions: + raise FileNotFoundError(f"Cannot find any installed {renderer} engines") + + # Find the path to the requested engine version or use default + engine_path = None if engine_version else all_versions[0]['path'] + if engine_version: + for ver in all_versions: + if ver['version'] == engine_version: + engine_path = ver['path'] + break + + # Download the required engine if not found locally + if not engine_path: + download_result = EngineManager.download_engine(renderer, engine_version) + if not download_result: + raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}") + engine_path = download_result['path'] + logger.info("Engine downloaded. Creating worker.") + + if not engine_path: + raise FileNotFoundError(f"Cannot find requested engine version {engine_version}") + + return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args, + parent=parent, name=name) + + if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') # print(EngineManager.newest_engine_version('blender', 'macos', 'arm64')) - EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a') - + # EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a') + EngineManager.engines_path = "/Users/brettwilliams/zordon-uploads/engines" + # print(EngineManager.is_version_downloaded("ffmpeg", "6.0")) + print(EngineManager.all_engines()) diff --git a/src/engines/ffmpeg/ffmpeg_downloader.py b/src/engines/ffmpeg/ffmpeg_downloader.py index f539a8e..3c36b49 100644 --- a/src/engines/ffmpeg/ffmpeg_downloader.py +++ b/src/engines/ffmpeg/ffmpeg_downloader.py @@ -4,14 +4,16 @@ import re import requests -from src.engines.core.downloader_core import download_and_extract_app +from src.engines.core.base_downloader import EngineDownloader +from src.engines.ffmpeg.ffmpeg_engine import FFMPEG from src.utilities.misc_helper import current_system_cpu, current_system_os logger = logging.getLogger() -supported_formats = ['.zip', '.tar.xz', '.dmg'] -class FFMPEGDownloader: +class FFMPEGDownloader(EngineDownloader): + + engine = FFMPEG # macOS FFMPEG mirror maintained by Evermeet - https://evermeet.cx/ffmpeg/ macos_url = "https://evermeet.cx/pub/ffmpeg/" @@ -88,17 +90,7 @@ class FFMPEGDownloader: return releases @classmethod - def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False): - try: - system_os = system_os or current_system_os() - cpu = cpu or current_system_cpu() - return cls.all_versions(system_os, cpu)[0] - except TypeError: - pass - return None - - @classmethod - def all_versions(cls, system_os=None, cpu=None): + def __all_versions(cls, system_os=None, cpu=None): system_os = system_os or current_system_os() cpu = cpu or current_system_cpu() versions_per_os = {'linux': cls.__get_linux_versions, 'macos': cls.__get_macos_versions, @@ -115,13 +107,6 @@ class FFMPEGDownloader: 'version': version}) return results - @classmethod - def version_is_available_to_download(cls, version, system_os=None, cpu=None): - for ver in cls.all_versions(system_os, cpu): - if ver['version'] == version: - return ver - return None - @classmethod def __get_remote_url_for_version(cls, version, system_os, cpu): # Platform specific naming cleanup @@ -141,13 +126,30 @@ class FFMPEGDownloader: logger.error("Unknown system os") return remote_url + @classmethod + def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False): + try: + system_os = system_os or current_system_os() + cpu = cpu or current_system_cpu() + return cls.__all_versions(system_os, cpu)[0] + except (IndexError, requests.exceptions.RequestException): + logger.error(f"Cannot get most recent version of ffmpeg") + return {} + + @classmethod + def version_is_available_to_download(cls, version, system_os=None, cpu=None): + for ver in cls.__all_versions(system_os, cpu): + if ver['version'] == version: + return ver + return None + @classmethod def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120): system_os = system_os or current_system_os() cpu = cpu or current_system_cpu() # Verify requested version is available - found_version = [item for item in cls.all_versions(system_os, cpu) if item['version'] == version] + found_version = [item for item in cls.__all_versions(system_os, cpu) if item['version'] == version] if not found_version: logger.error(f"Cannot find FFMPEG version {version} for {system_os} and {cpu}") return @@ -160,7 +162,7 @@ class FFMPEGDownloader: # Download and extract try: logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}") - download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout) + cls.__download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout) # naming cleanup to match existing naming convention output_path = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}') diff --git a/src/engines/ffmpeg/ffmpeg_engine.py b/src/engines/ffmpeg/ffmpeg_engine.py index b6147a2..1908f87 100644 --- a/src/engines/ffmpeg/ffmpeg_engine.py +++ b/src/engines/ffmpeg/ffmpeg_engine.py @@ -7,6 +7,16 @@ class FFMPEG(BaseRenderEngine): binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'} + @staticmethod + def downloader(): + from src.engines.ffmpeg.ffmpeg_downloader import FFMPEGDownloader + return FFMPEGDownloader + + @staticmethod + def worker_class(): + from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker + return FFMPEGRenderWorker + def version(self): version = None try: