From 7d1ecf1fa56ec1de7127dfbd70e1c7a42df4ee0d Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 20 Oct 2023 15:05:29 -0500 Subject: [PATCH] Downloadable engines (#34) * Add blender_downloader.py * Add engine_manager.py * Add additional methods to engine_manager.py * Refactor file layout to make engines on par with workers * Add system platform info to status response * Default to using system platform / cpu if none are provided * Add API to download an engine and some general cleanup * Add method to delete downloaded engine * Add API calls to download engines and delete downloads * Misc fixes --- requirements.txt | 4 +- src/api_server.py | 53 ++++- src/client/new_job_window.py | 4 +- src/{workers => }/engines/__init__.py | 0 src/{workers => }/engines/aerender_engine.py | 7 +- src/{workers => }/engines/base_engine.py | 16 +- src/{workers => }/engines/blender_engine.py | 34 ++- src/engines/engine_manager.py | 128 +++++++++++ src/{workers => }/engines/ffmpeg_engine.py | 29 +-- src/engines/scripts/__init__.py | 0 src/engines/scripts/blender/__init__.py | 0 .../scripts/blender/blender_downloader.py | 206 ++++++++++++++++++ .../engines/scripts/blender/get_file_info.py | 0 .../engines/scripts/blender/pack_project.py | 0 src/render_queue.py | 4 +- src/utilities/ffmpeg_helper.py | 8 +- src/workers/aerender_worker.py | 2 +- src/workers/base_worker.py | 9 +- src/workers/blender_worker.py | 6 +- src/workers/ffmpeg_worker.py | 8 +- src/{ => workers}/worker_factory.py | 0 21 files changed, 439 insertions(+), 79 deletions(-) rename src/{workers => }/engines/__init__.py (100%) rename src/{workers => }/engines/aerender_engine.py (78%) rename src/{workers => }/engines/base_engine.py (78%) rename src/{workers => }/engines/blender_engine.py (72%) create mode 100644 src/engines/engine_manager.py rename src/{workers => }/engines/ffmpeg_engine.py (59%) create mode 100644 src/engines/scripts/__init__.py create mode 100644 src/engines/scripts/blender/__init__.py create mode 100644 src/engines/scripts/blender/blender_downloader.py rename src/{workers => }/engines/scripts/blender/get_file_info.py (100%) rename src/{workers => }/engines/scripts/blender/pack_project.py (100%) rename src/{ => workers}/worker_factory.py (100%) diff --git a/requirements.txt b/requirements.txt index 3a9e33c..9a1a113 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,6 @@ json2html~=1.3.0 SQLAlchemy~=2.0.15 Pillow==9.5.0 zeroconf==0.64.1 -Pypubsub~=4.0.3 \ No newline at end of file +Pypubsub~=4.0.3 +tqdm==4.66.1 +dmglib \ No newline at end of file diff --git a/src/api_server.py b/src/api_server.py index 2f284af..b1474be 100755 --- a/src/api_server.py +++ b/src/api_server.py @@ -21,11 +21,12 @@ from flask import Flask, request, render_template, send_file, after_this_request from werkzeug.utils import secure_filename from src.distributed_job_manager import DistributedJobManager +from src.engines.engine_manager import EngineManager from src.render_queue import RenderQueue, JobNotFoundError from src.server_proxy import RenderServerProxy from src.utilities.server_helper import generate_thumbnail_for_job from src.utilities.zeroconf_server import ZeroconfServer -from src.worker_factory import RenderWorkerFactory +from src.workers.worker_factory import RenderWorkerFactory from src.workers.base_worker import string_to_status, RenderStatus logger = logging.getLogger() @@ -499,14 +500,21 @@ def clear_history(): def status(): renderer_data = {} for render_class in RenderWorkerFactory.supported_classes(): - if render_class.engine.renderer_path(): # only return renderers installed on host + if render_class.engine.default_renderer_path(): # only return renderers installed on host renderer_data[render_class.engine.name()] = \ - {'version': render_class.engine.version(), + {'versions': EngineManager.all_versions_for_engine(render_class.engine.name()), 'is_available': RenderQueue.is_available_for_job(render_class.engine.name()) } + # Get system info + system_platform = platform.system().lower().replace('darwin', 'macos') + system_platform_version = platform.mac_ver()[0] if system_platform == 'macos' else platform.release().lower() + system_cpu = platform.machine().lower().replace('amd64', 'x64') + return {"timestamp": datetime.now().isoformat(), - "platform": platform.platform(), + "system_platform": system_platform, + "system_platform_version": system_platform_version, + "system_cpu": system_cpu, "cpu_percent": psutil.cpu_percent(percpu=False), "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True), "cpu_count": psutil.cpu_count(logical=False), @@ -523,17 +531,36 @@ def status(): @server.get('/api/renderer_info') def renderer_info(): renderer_data = {} - for r in RenderWorkerFactory.supported_renderers(): - engine = RenderWorkerFactory.class_for_name(r).engine - if engine.renderer_path(): - renderer_data[r] = {'is_available': RenderQueue.is_available_for_job(engine.name()), - 'version': engine.version(), + for engine_name in RenderWorkerFactory.supported_renderers(): + engine = RenderWorkerFactory.class_for_name(engine_name).engine + if engine.default_renderer_path(): + + # Get all installed versions of engine + renderer_data[engine_name] = {'is_available': RenderQueue.is_available_for_job(engine.name()), + 'versions': EngineManager.all_versions_for_engine(engine_name), 'supported_extensions': engine.supported_extensions, - 'supported_export_formats': engine.get_output_formats(), - 'path': engine.renderer_path()} + 'supported_export_formats': engine().get_output_formats()} return renderer_data +@server.post('/api/download_engine') +def download_engine(): + download_result = EngineManager.download_engine(request.args.get('engine'), + request.args.get('version'), + request.args.get('system_os'), + request.args.get('cpu')) + return download_result if download_result else ("Error downloading requested engine", 500) + + +@server.post('/api/delete_engine') +def delete_engine_download(): + delete_result = EngineManager.delete_engine_download(request.args.get('engine'), + request.args.get('version'), + request.args.get('system_os'), + request.args.get('cpu')) + return "Success" if delete_result else ("Error deleting requested engine", 500) + + @server.route('/upload') def upload_file_page(): return render_template('upload.html', supported_renderers=RenderWorkerFactory.supported_renderers()) @@ -563,6 +590,10 @@ def start_server(background_thread=False): server.config['MAX_CONTENT_PATH'] = config['max_content_path'] server.config['enable_split_jobs'] = config.get('enable_split_jobs', False) + # Setup directory for saving engines to + EngineManager.engines_path = os.path.join(os.path.join(os.path.expanduser(config['upload_folder']), 'engines')) + os.makedirs(EngineManager.engines_path, exist_ok=True) + # disable most Flask logging flask_log = logging.getLogger('werkzeug') flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) diff --git a/src/client/new_job_window.py b/src/client/new_job_window.py index 19d80ac..27c5c18 100755 --- a/src/client/new_job_window.py +++ b/src/client/new_job_window.py @@ -224,7 +224,7 @@ class NewJobWindow(Frame): return if renderer == 'blender': - self.project_info = Blender.get_scene_info(self.chosen_file) + self.project_info = Blender().get_scene_info(self.chosen_file) self.draw_blender_settings() elif renderer == 'ffmpeg': f = FFMPEG.get_frame_count(self.chosen_file) @@ -371,7 +371,7 @@ class NewJobWindow(Frame): if renderer == 'blender': if self.blender_pack_textures.get(): self.progress_label.configure(text="Packing Blender file...") - new_path = Blender.pack_project_file(project_path=input_path, timeout=300) + new_path = Blender().pack_project_file(project_path=input_path, timeout=300) if new_path: logger.info(f"New Path is now {new_path}") input_path = new_path diff --git a/src/workers/engines/__init__.py b/src/engines/__init__.py similarity index 100% rename from src/workers/engines/__init__.py rename to src/engines/__init__.py diff --git a/src/workers/engines/aerender_engine.py b/src/engines/aerender_engine.py similarity index 78% rename from src/workers/engines/aerender_engine.py rename to src/engines/aerender_engine.py index e602038..fe335c6 100644 --- a/src/workers/engines/aerender_engine.py +++ b/src/engines/aerender_engine.py @@ -8,16 +8,15 @@ class AERender(BaseRenderEngine): supported_extensions = ['.aep'] - @classmethod - def version(cls): + def version(self): version = None try: - render_path = cls.renderer_path() + render_path = self.renderer_path() if render_path: ver_out = subprocess.check_output([render_path, '-version'], timeout=SUBPROCESS_TIMEOUT) version = ver_out.decode('utf-8').split(" ")[-1].strip() except Exception as e: - logger.error(f'Failed to get {cls.name()} version: {e}') + logger.error(f'Failed to get {self.name()} version: {e}') return version @classmethod diff --git a/src/workers/engines/base_engine.py b/src/engines/base_engine.py similarity index 78% rename from src/workers/engines/base_engine.py rename to src/engines/base_engine.py index 1d4d8c7..606d524 100644 --- a/src/workers/engines/base_engine.py +++ b/src/engines/base_engine.py @@ -11,12 +11,18 @@ class BaseRenderEngine(object): install_paths = [] supported_extensions = [] + def __init__(self, custom_path=None): + self.custom_renderer_path = custom_path + + def renderer_path(self): + return self.custom_renderer_path or self.default_renderer_path() + @classmethod def name(cls): return cls.__name__.lower() @classmethod - def renderer_path(cls): + def default_renderer_path(cls): path = None try: path = subprocess.check_output(['which', cls.name()], timeout=SUBPROCESS_TIMEOUT).decode('utf-8').strip() @@ -28,13 +34,11 @@ class BaseRenderEngine(object): logger.exception(e) return path - @classmethod - def version(cls): + def version(self): raise NotImplementedError("version not implemented") - @classmethod - def get_help(cls): - path = cls.renderer_path() + def get_help(self): + path = self.renderer_path() if not path: raise FileNotFoundError("renderer path not found") help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT, diff --git a/src/workers/engines/blender_engine.py b/src/engines/blender_engine.py similarity index 72% rename from src/workers/engines/blender_engine.py rename to src/engines/blender_engine.py index e15b778..c8bfae9 100644 --- a/src/workers/engines/blender_engine.py +++ b/src/engines/blender_engine.py @@ -14,11 +14,10 @@ class Blender(BaseRenderEngine): install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender'] supported_extensions = ['.blend'] - @classmethod - def version(cls): + def version(self): version = None try: - render_path = cls.renderer_path() + render_path = self.renderer_path() if render_path: ver_out = subprocess.check_output([render_path, '-v'], timeout=SUBPROCESS_TIMEOUT) version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip() @@ -26,17 +25,15 @@ class Blender(BaseRenderEngine): logger.error(f'Failed to get Blender version: {e}') return version - @classmethod - def get_output_formats(cls): - format_string = cls.get_help().split('Format Options')[-1].split('Animation Playback Options')[0] + def get_output_formats(self): + format_string = self.get_help().split('Format Options')[-1].split('Animation Playback Options')[0] formats = re.findall(r"'([A-Z_0-9]+)'", format_string) return formats - @classmethod - def run_python_expression(cls, project_path, python_expression, timeout=None): + def run_python_expression(self, project_path, python_expression, timeout=None): if os.path.exists(project_path): try: - return subprocess.run([cls.renderer_path(), '-b', project_path, '--python-expr', python_expression], + return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression], capture_output=True, timeout=timeout) except Exception as e: logger.warning(f"Error running python expression in blender: {e}") @@ -44,11 +41,10 @@ class Blender(BaseRenderEngine): else: raise FileNotFoundError(f'Project file not found: {project_path}') - @classmethod - def run_python_script(cls, project_path, script_path, timeout=None): + def run_python_script(self, project_path, script_path, timeout=None): if os.path.exists(project_path) and os.path.exists(script_path): try: - return subprocess.run([cls.renderer_path(), '-b', project_path, '--python', script_path], + return subprocess.run([self.default_renderer_path(), '-b', project_path, '--python', script_path], capture_output=True, timeout=timeout) except Exception as e: logger.warning(f"Error running python expression in blender: {e}") @@ -59,12 +55,11 @@ class Blender(BaseRenderEngine): raise FileNotFoundError(f'Python script not found: {script_path}') raise Exception("Uncaught exception") - @classmethod - def get_scene_info(cls, project_path, timeout=10): + def get_scene_info(self, project_path, timeout=10): scene_info = {} try: - results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'get_file_info.py'), timeout=timeout) + results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'get_file_info.py'), timeout=timeout) result_text = results.stdout.decode() for line in result_text.splitlines(): if line.startswith('SCENE_DATA:'): @@ -77,13 +72,12 @@ class Blender(BaseRenderEngine): logger.error(f'Error getting file details for .blend file: {e}') return scene_info - @classmethod - def pack_project_file(cls, project_path, timeout=30): + def pack_project_file(self, project_path, timeout=30): # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 try: logger.info(f"Starting to pack Blender file: {project_path}") - results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'pack_project.py'), timeout=timeout) + results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'pack_project.py'), timeout=timeout) result_text = results.stdout.decode() dir_name = os.path.dirname(project_path) diff --git a/src/engines/engine_manager.py b/src/engines/engine_manager.py new file mode 100644 index 0000000..dcd2f49 --- /dev/null +++ b/src/engines/engine_manager.py @@ -0,0 +1,128 @@ +import os +import logging +import platform +import shutil + +try: + from .blender_engine import Blender +except ImportError: + from blender_engine import Blender +try: + from .ffmpeg_engine import FFMPEG +except ImportError: + from ffmpeg_engine import FFMPEG + +logger = logging.getLogger() + + +class EngineManager: + + engines_path = "~/zordon-uploads/engines" + + @classmethod + def supported_engines(cls): + return [Blender, FFMPEG] + + @classmethod + def all_engines(cls): + results = [] + # Parse downloaded engine directory + try: + all_items = os.listdir(cls.engines_path) + all_directories = [item for item in all_items if os.path.isdir(os.path.join(cls.engines_path, item))] + + for directory in all_directories: + # Split the input string by dashes to get segments + segments = directory.split('-') + + # Define the keys for each word + keys = ["engine", "version", "system_os", "cpu"] + + # Create a dictionary with named keys + executable_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender.app'} + result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))} + result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names[result_dict['system_os']]) + result_dict['type'] = 'managed' + results.append(result_dict) + except FileNotFoundError: + logger.warning("Cannot find local engines download directory") + + # add system installs to this list + for eng in cls.supported_engines(): + if eng.default_renderer_path(): + results.append({'engine': eng.name(), 'version': eng().version(), + 'system_os': cls.system_os(), + 'cpu': cls.system_cpu(), + 'path': eng.default_renderer_path(), 'type': 'system'}) + + return results + + @classmethod + def all_versions_for_engine(cls, engine): + return [x for x in cls.all_engines() if x['engine'] == engine] + + @classmethod + def newest_engine_version(cls, engine, system_os=None, cpu=None): + system_os = system_os or cls.system_os() + cpu = cpu or cls.system_cpu() + + try: + filtered = [x for x in cls.all_engines() if x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu] + versions = sorted(filtered, key=lambda x: x['version'], reverse=True) + return versions[0] + except IndexError: + logger.error(f"Cannot find newest engine version for {engine}-{system_os}-{cpu}") + return None + + @classmethod + def has_engine_version(cls, engine, version, system_os=None, cpu=None): + system_os = system_os or cls.system_os() + cpu = cpu or cls.system_cpu() + + filtered = [x for x in cls.all_engines() if + x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu and x['version'] == version] + return filtered[0] if filtered else False + + @staticmethod + def system_os(): + return platform.system().lower().replace('darwin', 'macos') + + @staticmethod + def system_cpu(): + return platform.machine().lower().replace('amd64', 'x64') + + @classmethod + def download_engine(cls, engine, version, system_os=None, cpu=None): + existing_download = cls.has_engine_version(engine, version, system_os, cpu) + if existing_download: + logger.info(f"Requested download of {engine} {version}, but local copy already exists") + return existing_download + + if engine == "blender": + from .scripts.blender.blender_downloader import BlenderDownloader + logger.info(f"Requesting download of {engine} {version}") + if BlenderDownloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu): + return cls.has_engine_version(engine, version, system_os, cpu) + else: + logger.error("Error downloading Engine") + + return None # Return None to indicate an error + + @classmethod + def delete_engine_download(cls, engine, version, system_os=None, cpu=None): + logger.info(f"Requested deletion of engine: {engine}-{version}") + found = cls.has_engine_version(engine, version, system_os, cpu) + if found: + dir_path = os.path.dirname(found['path']) + shutil.rmtree(dir_path, ignore_errors=True) + logger.info(f"Engine {engine}-{version}-{found['system_os']}-{found['cpu']} successfully deleted") + return True + else: + logger.error(f"Cannot find engine: {engine}-{version}") + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # print(EngineManager.newest_engine_version('blender', 'macos', 'arm64')) + EngineManager.delete_engine_download('blender', '3.2.1', 'windows', 'x64') diff --git a/src/workers/engines/ffmpeg_engine.py b/src/engines/ffmpeg_engine.py similarity index 59% rename from src/workers/engines/ffmpeg_engine.py rename to src/engines/ffmpeg_engine.py index 6990a64..eb46c2b 100644 --- a/src/workers/engines/ffmpeg_engine.py +++ b/src/engines/ffmpeg_engine.py @@ -7,11 +7,10 @@ import re class FFMPEG(BaseRenderEngine): - @classmethod - def version(cls): + def version(self): version = None try: - ver_out = subprocess.check_output([cls.renderer_path(), '-version'], + ver_out = subprocess.check_output([self.renderer_path(), '-version'], timeout=SUBPROCESS_TIMEOUT).decode('utf-8') match = re.match(".*version\s*(\S+)\s*Copyright", ver_out) if match: @@ -20,29 +19,25 @@ class FFMPEG(BaseRenderEngine): logger.error("Failed to get FFMPEG version: {}".format(e)) return version - @classmethod - def get_encoders(cls): - raw_stdout = subprocess.check_output([cls.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL, - timeout=SUBPROCESS_TIMEOUT).decode('utf-8') + def get_encoders(self): + raw_stdout = subprocess.check_output([self.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL, + timeout=SUBPROCESS_TIMEOUT).decode('utf-8') pattern = '(?P[VASFXBD.]{6})\s+(?P\S{2,})\s+(?P.*)' encoders = [m.groupdict() for m in re.finditer(pattern, raw_stdout)] return encoders - @classmethod - def get_all_formats(cls): - raw_stdout = subprocess.check_output([cls.renderer_path(), '-formats'], stderr=subprocess.DEVNULL, - timeout=SUBPROCESS_TIMEOUT).decode('utf-8') + def get_all_formats(self): + raw_stdout = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL, + timeout=SUBPROCESS_TIMEOUT).decode('utf-8') pattern = '(?P[DE]{1,2})\s+(?P\S{2,})\s+(?P.*)' formats = [m.groupdict() for m in re.finditer(pattern, raw_stdout)] return formats - @classmethod - def get_output_formats(cls): - return [x for x in cls.get_all_formats() if 'E' in x['type'].upper()] + def get_output_formats(self): + return [x for x in self.get_all_formats() if 'E' in x['type'].upper()] - @classmethod - def get_frame_count(cls, path_to_file): - raw_stdout = subprocess.check_output([cls.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', + def get_frame_count(self, path_to_file): + raw_stdout = subprocess.check_output([self.default_renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', '-f', 'null', '-'], stderr=subprocess.STDOUT, timeout=SUBPROCESS_TIMEOUT).decode('utf-8') match = re.findall(r'frame=\s*(\d+)', raw_stdout) diff --git a/src/engines/scripts/__init__.py b/src/engines/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/engines/scripts/blender/__init__.py b/src/engines/scripts/blender/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/engines/scripts/blender/blender_downloader.py b/src/engines/scripts/blender/blender_downloader.py new file mode 100644 index 0000000..832fca4 --- /dev/null +++ b/src/engines/scripts/blender/blender_downloader.py @@ -0,0 +1,206 @@ +import logging +import os +import re +import platform +import shutil +import tarfile +import tempfile +import zipfile + +import requests +from tqdm import tqdm + +# url = "https://download.blender.org/release/" +url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing + +logger = logging.getLogger() +supported_formats = ['.zip', '.tar.xz', '.dmg'] + + +class BlenderDownloader: + + @staticmethod + def get_major_versions(): + try: + response = requests.get(url) + response.raise_for_status() + + # Use regex to find all the tags and extract the href attribute + link_pattern = r'Blender(\d+[^<]+)' + link_matches = re.findall(link_pattern, response.text) + + major_versions = [link[-1].strip('/') for link in link_matches] + major_versions.sort(reverse=True) + return major_versions + except requests.exceptions.RequestException as e: + logger.error(f"Error: {e}") + return None + + @staticmethod + def get_minor_versions(major_version, system_os=None, cpu=None): + + base_url = url + 'Blender' + major_version + + response = requests.get(base_url) + response.raise_for_status() + + versions_pattern = r'blender-(?P[\d\.]+)-(?P\w+)-(?P\w+).*' + versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)] + + # Filter to just the supported formats + versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)] + + if system_os: + versions_data = [x for x in versions_data if x['system_os'] == system_os] + if cpu: + versions_data = [x for x in versions_data if x['cpu'] == cpu] + + for v in versions_data: + v['url'] = os.path.join(base_url, v['file']) + + versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) + return versions_data + + @staticmethod + def find_LTS_versions(): + response = requests.get('https://www.blender.org/download/lts/') + response.raise_for_status() + + lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/' + lts_matches = re.findall(lts_pattern, response.text) + lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))] + lts_versions.sort(reverse=True) + + return lts_versions + + @classmethod + def find_most_recent_version(cls, system_os, cpu, lts_only=False): + try: + major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0] + most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0] + return most_recent + except IndexError: + logger.error("Cannot find a most recent version") + + @classmethod + def download_engine(cls, version, download_location, system_os=None, cpu=None): + system_os = system_os or platform.system().lower().replace('darwin', 'macos') + cpu = cpu or platform.machine().lower().replace('amd64', 'x64') + + try: + logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}") + major_version = '.'.join(version.split('.')[:2]) + minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version] + # we get the URL instead of calculating it ourselves. May change this + + cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location) + except IndexError: + logger.error("Cannot find requested engine") + + @classmethod + def download_and_extract_app(cls, remote_url, download_location): + + binary_path = None + + # Create a temp download directory + temp_download_dir = tempfile.mkdtemp() + downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url)) + + try: + output_dir_name = os.path.basename(remote_url) + for fmt in supported_formats: + output_dir_name = output_dir_name.split(fmt)[0] + + if os.path.exists(os.path.join(download_location, output_dir_name)): + logger.error(f"Engine download for {output_dir_name} already exists") + return + + if not os.path.exists(downloaded_file_path): + # Make a GET request to the URL with stream=True to enable streaming + logger.info(f"Downloading {output_dir_name} from {remote_url}") + response = requests.get(remote_url, stream=True) + + # Check if the request was successful + if response.status_code == 200: + # Get the total file size from the "Content-Length" header + file_size = int(response.headers.get("Content-Length", 0)) + + # Create a progress bar using tqdm + progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) + + # Open a file for writing in binary mode + with open(downloaded_file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # Write the chunk to the file + file.write(chunk) + # Update the progress bar + progress_bar.update(len(chunk)) + + # Close the progress bar + progress_bar.close() + logger.info(f"Successfully downloaded {os.path.basename(downloaded_file_path)}") + else: + logger.error(f"Failed to download the file. Status code: {response.status_code}") + + os.makedirs(download_location, exist_ok=True) + + # Extract the downloaded Blender file + # Linux - Process .tar.xz files + if downloaded_file_path.lower().endswith('.tar.xz'): + try: + with tarfile.open(downloaded_file_path, 'r:xz') as tar: + tar.extractall(path=download_location) + os.path.join(download_location, output_dir_name, 'blender') + logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}') + except tarfile.TarError as e: + logger.error(f'Error extracting {downloaded_file_path}: {e}') + except FileNotFoundError: + logger.error(f'File not found: {downloaded_file_path}') + + # Windows - Process .zip files + elif downloaded_file_path.lower().endswith('.zip'): + try: + with zipfile.ZipFile(downloaded_file_path, 'r') as zip_ref: + zip_ref.extractall(download_location) + logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}') + except zipfile.BadZipFile as e: + logger.error(f'Error: {downloaded_file_path} is not a valid ZIP file.') + except FileNotFoundError: + logger.error(f'File not found: {downloaded_file_path}') + + # macOS - Process .dmg files + elif downloaded_file_path.lower().endswith('.dmg'): + import dmglib + dmg = dmglib.DiskImage(downloaded_file_path) + for mount_point in dmg.attach(): + try: + # Copy the entire .app bundle to the destination directory + shutil.copytree(os.path.join(mount_point, 'Blender.app'), + os.path.join(download_location, output_dir_name, 'Blender.app')) + binary_path = os.path.join(download_location, output_dir_name, 'Blender.app') + logger.info(f'Successfully copied {os.path.basename(downloaded_file_path)} to {download_location}') + except FileNotFoundError: + logger.error(f'Error: The source .app bundle does not exist.') + except PermissionError: + logger.error(f'Error: Permission denied to copy {download_location}.') + except Exception as e: + logger.error(f'An error occurred: {e}') + dmg.detach() + + else: + logger.error("Unknown file. Unable to extract binary.") + + except Exception as e: + logger.exception(e) + + # remove downloaded file on completion + shutil.rmtree(temp_download_dir) + return binary_path + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + BlenderDownloader.download_engine('3.3.1', download_location="/Users/brett/Desktop/test/releases", system_os='linux', cpu='x64') + diff --git a/src/workers/engines/scripts/blender/get_file_info.py b/src/engines/scripts/blender/get_file_info.py similarity index 100% rename from src/workers/engines/scripts/blender/get_file_info.py rename to src/engines/scripts/blender/get_file_info.py diff --git a/src/workers/engines/scripts/blender/pack_project.py b/src/engines/scripts/blender/pack_project.py similarity index 100% rename from src/workers/engines/scripts/blender/pack_project.py rename to src/engines/scripts/blender/pack_project.py diff --git a/src/render_queue.py b/src/render_queue.py index 9e6d535..f8f3c4a 100755 --- a/src/render_queue.py +++ b/src/render_queue.py @@ -5,7 +5,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.utilities.status_utils import RenderStatus -from src.worker_factory import RenderWorkerFactory +from src.workers.worker_factory import RenderWorkerFactory from src.workers.base_worker import Base logger = logging.getLogger() @@ -100,7 +100,7 @@ class RenderQueue: @classmethod def is_available_for_job(cls, renderer, priority=2): - if not RenderWorkerFactory.class_for_name(renderer).engine.renderer_path(): + if not RenderWorkerFactory.class_for_name(renderer).engine.default_renderer_path(): return False instances = cls.renderer_instances() diff --git a/src/utilities/ffmpeg_helper.py b/src/utilities/ffmpeg_helper.py index 96bc2df..941c383 100644 --- a/src/utilities/ffmpeg_helper.py +++ b/src/utilities/ffmpeg_helper.py @@ -1,20 +1,20 @@ import subprocess -from src.workers.engines.ffmpeg_engine import FFMPEG +from src.engines.ffmpeg_engine import FFMPEG def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4, start_frame=1): - subprocess.run([FFMPEG.renderer_path(), "-framerate", str(framerate), "-start_number", str(start_frame), "-i", + subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number", str(start_frame), "-i", f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile), '-pix_fmt', 'yuva444p10le', output_path], check=True) def save_first_frame(source_path, dest_path, max_width=1280): - subprocess.run([FFMPEG.renderer_path(), '-i', source_path, '-vf', f'scale={max_width}:-1', + subprocess.run([FFMPEG.default_renderer_path(), '-i', source_path, '-vf', f'scale={max_width}:-1', '-vframes', '1', dest_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) def generate_thumbnail(source_path, dest_path, max_width=240, fps=12): - subprocess.run([FFMPEG.renderer_path(), '-i', source_path, '-vf', + subprocess.run([FFMPEG.default_renderer_path(), '-i', source_path, '-vf', f"scale={max_width}:trunc(ow/a/2)*2,format=yuv420p", '-r', str(fps), '-c:v', 'libx264', '-preset', 'ultrafast', '-an', dest_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) diff --git a/src/workers/aerender_worker.py b/src/workers/aerender_worker.py index f475bb7..d9e491e 100644 --- a/src/workers/aerender_worker.py +++ b/src/workers/aerender_worker.py @@ -7,7 +7,7 @@ import re import time from src.workers.base_worker import BaseRenderWorker, timecode_to_frames -from src.workers.engines.aerender_engine import AERender +from src.engines.aerender_engine import AERender def aerender_path(): diff --git a/src/workers/base_worker.py b/src/workers/base_worker.py index c6ccc83..8f52c58 100644 --- a/src/workers/base_worker.py +++ b/src/workers/base_worker.py @@ -64,7 +64,8 @@ class BaseRenderWorker(Base): self.args = args or {} self.date_created = datetime.now() self.renderer = self.engine.name() - self.renderer_version = self.engine.version() + self.renderer_version = self.engine().version() + self.custom_renderer_path = None self.priority = priority self.parent = parent self.children = {} @@ -158,7 +159,7 @@ class BaseRenderWorker(Base): self.errors.append(msg) return - if not self.engine.renderer_path(): + if not self.engine.default_renderer_path() and not self.custom_renderer_path: self.status = RenderStatus.ERROR msg = 'Cannot find render engine path for {}'.format(self.engine.name()) logger.error(msg) @@ -167,7 +168,7 @@ class BaseRenderWorker(Base): self.status = RenderStatus.RUNNING self.start_time = datetime.now() - logger.info(f'Starting {self.engine.name()} {self.engine.version()} Render for {self.input_path} | ' + logger.info(f'Starting {self.engine.name()} {self.engine().version()} Render for {self.input_path} | ' f'Frame Count: {self.total_frames}') self.__thread.start() @@ -182,7 +183,7 @@ class BaseRenderWorker(Base): with open(self.log_path(), "a") as f: - f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine.version()} " + f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine().version()} " f"render for {self.input_path}\n\n") f.write(f"Running command: {subprocess_cmds}\n") f.write('=' * 80 + '\n\n') diff --git a/src/workers/blender_worker.py b/src/workers/blender_worker.py index 72ac829..20fa943 100644 --- a/src/workers/blender_worker.py +++ b/src/workers/blender_worker.py @@ -2,7 +2,7 @@ import re from collections import Counter -from src.workers.engines.blender_engine import Blender +from src.engines.blender_engine import Blender from src.utilities.ffmpeg_helper import image_sequence_to_video from src.workers.base_worker import * @@ -24,7 +24,7 @@ class BlenderRenderWorker(BaseRenderWorker): self.__frame_percent_complete = 0.0 # Scene Info - self.scene_info = Blender.get_scene_info(input_path) + self.scene_info = Blender().get_scene_info(input_path) self.start_frame = int(self.scene_info.get('start_frame', 1)) self.end_frame = int(self.scene_info.get('end_frame', self.start_frame)) self.project_length = (self.end_frame - self.start_frame) + 1 @@ -32,7 +32,7 @@ class BlenderRenderWorker(BaseRenderWorker): def generate_worker_subprocess(self): - cmd = [self.engine.renderer_path()] + cmd = [self.engine.default_renderer_path()] if self.args.get('background', True): # optionally run render not in background cmd.append('-b') cmd.append(self.input_path) diff --git a/src/workers/ffmpeg_worker.py b/src/workers/ffmpeg_worker.py index 2287ee1..218ce83 100644 --- a/src/workers/ffmpeg_worker.py +++ b/src/workers/ffmpeg_worker.py @@ -3,7 +3,7 @@ import re import subprocess from src.workers.base_worker import BaseRenderWorker -from src.workers.engines.ffmpeg_engine import FFMPEG +from src.engines.ffmpeg_engine import FFMPEG class FFMPEGRenderWorker(BaseRenderWorker): @@ -14,8 +14,8 @@ class FFMPEGRenderWorker(BaseRenderWorker): super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name) - stream_info = subprocess.check_output([self.engine.renderer_path(), "-i", # https://stackoverflow.com/a/61604105 - input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y", + stream_info = subprocess.check_output([self.engine.default_renderer_path(), "-i", # https://stackoverflow.com/a/61604105 + input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y", "/dev/null"], stderr=subprocess.STDOUT).decode('utf-8') found_frames = re.findall('frame=\s*(\d+)', stream_info) self.project_length = found_frames[-1] if found_frames else '-1' @@ -23,7 +23,7 @@ class FFMPEGRenderWorker(BaseRenderWorker): def generate_worker_subprocess(self): - cmd = [self.engine.renderer_path(), '-y', '-stats', '-i', self.input_path] + cmd = [self.engine.default_renderer_path(), '-y', '-stats', '-i', self.input_path] # Resize frame if self.args.get('x_resolution', None) and self.args.get('y_resolution', None): diff --git a/src/worker_factory.py b/src/workers/worker_factory.py similarity index 100% rename from src/worker_factory.py rename to src/workers/worker_factory.py