diff --git a/lib/engines/blender_engine.py b/lib/engines/blender_engine.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/engines/ffmpeg_engine.py b/lib/engines/ffmpeg_engine.py deleted file mode 100644 index 8d61033..0000000 --- a/lib/engines/ffmpeg_engine.py +++ /dev/null @@ -1,4 +0,0 @@ - - -if __name__ == "__main__": - print(FFMPEG.get_frame_count('/Users/brett/Desktop/Big_Fire_02.mov')) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d70e9a8..fc5656a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ requests==2.31.0 +requests_toolbelt==1.0.0 psutil==5.9.6 PyYAML==6.0.1 Flask==3.0.0 diff --git a/src/api_server.py b/src/api_server.py index 4f69dc3..209b8fd 100755 --- a/src/api_server.py +++ b/src/api_server.py @@ -7,6 +7,7 @@ import platform import shutil import socket import ssl +import tempfile import threading import time import zipfile @@ -26,6 +27,7 @@ from src.render_queue import RenderQueue, JobNotFoundError from src.server_proxy import RenderServerProxy from src.utilities.server_helper import generate_thumbnail_for_job from src.utilities.zeroconf_server import ZeroconfServer +from src.utilities.misc_helper import system_safe_path from src.workers.worker_factory import RenderWorkerFactory from src.workers.base_worker import string_to_status, RenderStatus @@ -54,7 +56,7 @@ def sorted_jobs(all_jobs, sort_by_date=True): @server.route('/') @server.route('/index') def index(): - with open('config/presets.yaml') as f: + with open(system_safe_path('config/presets.yaml')) as f: render_presets = yaml.load(f, Loader=yaml.FullLoader) return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()), @@ -188,7 +190,7 @@ def get_job_status(job_id): @server.get('/api/job//logs') def get_job_logs(job_id): found_job = RenderQueue.job_with_id(job_id) - log_path = found_job.log_path() + log_path = system_safe_path(found_job.log_path()), log_data = None if log_path and os.path.exists(log_path): with open(log_path) as file: @@ -232,10 +234,11 @@ def download_all(job_id): found_job = RenderQueue.job_with_id(job_id) output_dir = os.path.dirname(found_job.output_path) if os.path.exists(output_dir): - zip_filename = os.path.join('/tmp', pathlib.Path(found_job.input_path).stem + '.zip') + zip_filename = system_safe_path(os.path.join(tempfile.gettempdir(), + pathlib.Path(found_job.input_path).stem + '.zip')) with ZipFile(zip_filename, 'w') as zipObj: for f in os.listdir(output_dir): - zipObj.write(filename=os.path.join(output_dir, f), + zipObj.write(filename=system_safe_path(os.path.join(output_dir, f)), arcname=os.path.basename(f)) return send_file(zip_filename, mimetype="zip", as_attachment=True, ) else: @@ -244,7 +247,8 @@ def download_all(job_id): @server.get('/api/presets') def presets(): - with open('config/presets.yaml') as f: + presets_path = system_safe_path('config/presets.yaml') + with open(presets_path) as f: presets = yaml.load(f, Loader=yaml.FullLoader) return presets @@ -387,7 +391,7 @@ def add_job_handler(): try: # prepare output paths output_dir = os.path.join(job_dir, job_data.get('name') if len(jobs_list) > 1 else 'output') - os.makedirs(output_dir, exist_ok=True) + os.makedirs(system_safe_path(output_dir), exist_ok=True) # get new output path in output_dir job_data['output_path'] = os.path.join(output_dir, os.path.basename( @@ -398,6 +402,7 @@ def add_job_handler(): worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'], input_path=loaded_project_local_path, output_path=job_data["output_path"], + engine_version=job_data.get('engine_version'), args=job_data.get('args', {})) worker.status = job_data.get("initial_status", worker.status) worker.parent = job_data.get("parent", worker.parent) @@ -500,7 +505,7 @@ def clear_history(): def status(): renderer_data = {} for render_class in RenderWorkerFactory.supported_classes(): - if render_class.engine.default_renderer_path(): # only return renderers installed on host + if EngineManager.all_versions_for_engine(render_class.name): # only return renderers installed on host renderer_data[render_class.engine.name()] = \ {'versions': EngineManager.all_versions_for_engine(render_class.engine.name()), 'is_available': RenderQueue.is_available_for_job(render_class.engine.name()) @@ -533,13 +538,15 @@ def renderer_info(): renderer_data = {} for engine_name in RenderWorkerFactory.supported_renderers(): engine = RenderWorkerFactory.class_for_name(engine_name).engine - if engine.default_renderer_path(): - # Get all installed versions of engine + # Get all installed versions of engine + installed_versions = EngineManager.all_versions_for_engine(engine_name) + if installed_versions: + install_path = installed_versions[0]['path'] renderer_data[engine_name] = {'is_available': RenderQueue.is_available_for_job(engine.name()), - 'versions': EngineManager.all_versions_for_engine(engine_name), + 'versions': installed_versions, 'supported_extensions': engine.supported_extensions, - 'supported_export_formats': engine().get_output_formats()} + 'supported_export_formats': engine(install_path).get_output_formats()} return renderer_data @@ -581,7 +588,7 @@ def start_server(background_thread=False): RenderQueue.evaluate_queue() time.sleep(delay_sec) - with open('config/config.yaml') as f: + with open(system_safe_path('config/config.yaml')) as f: config = yaml.load(f, Loader=yaml.FullLoader) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S', @@ -594,15 +601,20 @@ def start_server(background_thread=False): # load flask settings server.config['HOSTNAME'] = local_hostname server.config['PORT'] = int(config.get('port_number', 8080)) - server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder']) - server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs') + server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(config['upload_folder'])) + server.config['THUMBS_FOLDER'] = system_safe_path(os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs')) server.config['MAX_CONTENT_PATH'] = config['max_content_path'] server.config['enable_split_jobs'] = config.get('enable_split_jobs', False) # Setup directory for saving engines to - EngineManager.engines_path = os.path.join(os.path.join(os.path.expanduser(config['upload_folder']), 'engines')) + EngineManager.engines_path = system_safe_path(os.path.join(os.path.join(os.path.expanduser(config['upload_folder']), 'engines'))) os.makedirs(EngineManager.engines_path, exist_ok=True) + # Debug info + logger.debug(f"Upload directory: {server.config['UPLOAD_FOLDER']}") + logger.debug(f"Thumbs directory: {server.config['THUMBS_FOLDER']}") + logger.debug(f"Engines directory: {EngineManager.engines_path}") + # disable most Flask logging flask_log = logging.getLogger('werkzeug') flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) diff --git a/src/engines/base_engine.py b/src/engines/base_engine.py index 9d31b98..ccb3b07 100644 --- a/src/engines/base_engine.py +++ b/src/engines/base_engine.py @@ -13,6 +13,8 @@ class BaseRenderEngine(object): def __init__(self, custom_path=None): self.custom_renderer_path = custom_path + if not self.renderer_path(): + raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance") def renderer_path(self): return self.custom_renderer_path or self.default_renderer_path() @@ -24,9 +26,9 @@ class BaseRenderEngine(object): @classmethod def default_renderer_path(cls): path = None - try: + try: # Linux and macOS path = subprocess.check_output(['which', cls.name()], timeout=SUBPROCESS_TIMEOUT).decode('utf-8').strip() - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): for p in cls.install_paths: if os.path.exists(p): path = p diff --git a/src/engines/blender_engine.py b/src/engines/blender_engine.py index 0868ecf..6d3ab39 100644 --- a/src/engines/blender_engine.py +++ b/src/engines/blender_engine.py @@ -13,6 +13,7 @@ class Blender(BaseRenderEngine): install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender'] supported_extensions = ['.blend'] + binary_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender'} def version(self): version = None @@ -36,18 +37,17 @@ class Blender(BaseRenderEngine): return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression], capture_output=True, timeout=timeout) except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") - pass + logger.error(f"Error running python expression in blender: {e}") else: raise FileNotFoundError(f'Project file not found: {project_path}') def run_python_script(self, project_path, script_path, timeout=None): if os.path.exists(project_path) and os.path.exists(script_path): try: - return subprocess.run([self.default_renderer_path(), '-b', project_path, '--python', script_path], + return subprocess.run([self.renderer_path(), '-b', project_path, '--python', script_path], capture_output=True, timeout=timeout) except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") + logger.warning(f"Error running python script in blender: {e}") pass elif not os.path.exists(project_path): raise FileNotFoundError(f'Project file not found: {project_path}') @@ -58,8 +58,9 @@ class Blender(BaseRenderEngine): def get_scene_info(self, project_path, timeout=10): scene_info = {} try: - results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'get_file_info.py'), timeout=timeout) + script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'get_file_info.py') + results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout) result_text = results.stdout.decode() for line in result_text.splitlines(): if line.startswith('SCENE_DATA:'): @@ -76,8 +77,9 @@ class Blender(BaseRenderEngine): # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 try: logger.info(f"Starting to pack Blender file: {project_path}") - results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'pack_project.py'), timeout=timeout) + script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'pack_project.py') + results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout) result_text = results.stdout.decode() dir_name = os.path.dirname(project_path) diff --git a/src/engines/downloaders/blender_downloader.py b/src/engines/downloaders/blender_downloader.py index 3899077..beda93a 100644 --- a/src/engines/downloaders/blender_downloader.py +++ b/src/engines/downloaders/blender_downloader.py @@ -53,7 +53,7 @@ class BlenderDownloader: versions_data = [x for x in versions_data if x['cpu'] == cpu] for v in versions_data: - v['url'] = os.path.join(base_url, v['file']) + v['url'] = base_url + '/' + v['file'] versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) return versions_data diff --git a/src/engines/downloaders/ffmpeg_downloader.py b/src/engines/downloaders/ffmpeg_downloader.py index a5ecb66..32aa4d4 100644 --- a/src/engines/downloaders/ffmpeg_downloader.py +++ b/src/engines/downloaders/ffmpeg_downloader.py @@ -87,7 +87,7 @@ class FFMPEGDownloader: release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases' remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz') elif system_os == 'windows': - remote_url = os.path.join(cls.windows_download_url, version, f'ffmpeg-{version}-full_build.zip') + remote_url = f"{cls.windows_download_url.strip('/')}/{version}/ffmpeg-{version}-full_build.zip" # Download and extract try: diff --git a/src/engines/engine_manager.py b/src/engines/engine_manager.py index c999747..7c7106a 100644 --- a/src/engines/engine_manager.py +++ b/src/engines/engine_manager.py @@ -4,6 +4,7 @@ import platform import shutil from .downloaders.blender_downloader import BlenderDownloader from .downloaders.ffmpeg_downloader import FFMPEGDownloader +from ..utilities.misc_helper import system_safe_path try: from .blender_engine import Blender @@ -37,14 +38,25 @@ class EngineManager: # Split the input string by dashes to get segments segments = directory.split('-') - # Define the keys for each word - keys = ["engine", "version", "system_os", "cpu"] - # Create a dictionary with named keys - executable_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender.app'} + keys = ["engine", "version", "system_os", "cpu"] result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))} - result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names.get(result_dict['system_os'], 'unknown')) result_dict['type'] = 'managed' + + # Figure out the binary name for the path + binary_name = result_dict['engine'].lower() + for eng in cls.supported_engines(): + if eng.name().lower() == result_dict['engine']: + binary_name = eng.binary_names.get(result_dict['system_os'], binary_name) + + # Find path to binary + path = None + for root, _, files in os.walk(system_safe_path(os.path.join(cls.engines_path, directory))): + if binary_name in files: + path = os.path.join(root, binary_name) + break + + result_dict['path'] = path results.append(result_dict) except FileNotFoundError: logger.warning("Cannot find local engines download directory") @@ -113,11 +125,15 @@ class EngineManager: return # Get the appropriate downloader class based on the engine type - downloader = downloader_classes[engine] - if downloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu): - return cls.has_engine_version(engine, version, system_os, cpu) - else: + downloader_classes[engine].download_engine(version, download_location=cls.engines_path, + system_os=system_os, cpu=cpu) + + # Check that engine was properly downloaded + found_engine = cls.has_engine_version(engine, version, system_os, cpu) + if not found_engine: logger.error(f"Error downloading {engine}") + return found_engine + @classmethod def delete_engine_download(cls, engine, version, system_os=None, cpu=None): diff --git a/src/engines/ffmpeg_engine.py b/src/engines/ffmpeg_engine.py index 8dec66e..52e59b5 100644 --- a/src/engines/ffmpeg_engine.py +++ b/src/engines/ffmpeg_engine.py @@ -7,6 +7,8 @@ import re class FFMPEG(BaseRenderEngine): + binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'} + def version(self): version = None try: @@ -31,11 +33,15 @@ class FFMPEG(BaseRenderEngine): return encoders def get_all_formats(self): - formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL, - timeout=SUBPROCESS_TIMEOUT).decode('utf-8') - pattern = '(?P[DE]{1,2})\s+(?P\S{2,})\s+(?P.*)' - all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)] - return all_formats + try: + formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL, + timeout=SUBPROCESS_TIMEOUT).decode('utf-8') + pattern = '(?P[DE]{1,2})\s+(?P\S{2,})\s+(?P.*)\r' + all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)] + return all_formats + except Exception as e: + logger.error(f"Error getting all formats: {e}") + return [] def extension_for_format(self, ffmpeg_format): # Extract the common extension using regex @@ -53,7 +59,7 @@ class FFMPEG(BaseRenderEngine): return [x for x in self.get_all_formats() if 'E' in x['type'].upper()] def get_frame_count(self, path_to_file): - raw_stdout = subprocess.check_output([self.default_renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', + raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', '-f', 'null', '-'], stderr=subprocess.STDOUT, timeout=SUBPROCESS_TIMEOUT).decode('utf-8') match = re.findall(r'frame=\s*(\d+)', raw_stdout) diff --git a/src/render_queue.py b/src/render_queue.py index f8f3c4a..d8f2bda 100755 --- a/src/render_queue.py +++ b/src/render_queue.py @@ -5,7 +5,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.utilities.status_utils import RenderStatus -from src.workers.worker_factory import RenderWorkerFactory +from src.engines.engine_manager import EngineManager from src.workers.base_worker import Base logger = logging.getLogger() @@ -100,7 +100,8 @@ class RenderQueue: @classmethod def is_available_for_job(cls, renderer, priority=2): - if not RenderWorkerFactory.class_for_name(renderer).engine.default_renderer_path(): + + if not EngineManager.all_versions_for_engine(renderer): return False instances = cls.renderer_instances() diff --git a/src/utilities/misc_helper.py b/src/utilities/misc_helper.py index 15821e1..0362e31 100644 --- a/src/utilities/misc_helper.py +++ b/src/utilities/misc_helper.py @@ -1,5 +1,6 @@ import logging import os +import platform import subprocess from datetime import datetime @@ -103,3 +104,9 @@ def get_file_size_human(file_path): else: return f"{size_in_bytes / 1024 ** 4:.2f} TB" + +# Convert path to the appropriate format for the current platform +def system_safe_path(path): + if platform.system().lower() == "windows": + return os.path.normpath(path) + return path.replace("\\", "/") diff --git a/src/workers/base_worker.py b/src/workers/base_worker.py index 8f52c58..44eebfa 100644 --- a/src/workers/base_worker.py +++ b/src/workers/base_worker.py @@ -30,6 +30,7 @@ class BaseRenderWorker(Base): end_time = Column(DateTime, nullable=True) renderer = Column(String) renderer_version = Column(String) + renderer_path = Column(String) priority = Column(Integer) project_length = Column(Integer) start_frame = Column(Integer) @@ -42,7 +43,7 @@ class BaseRenderWorker(Base): engine = None - def __init__(self, input_path, output_path, priority=2, args=None, ignore_extensions=True, parent=None, + def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None, name=None): if not ignore_extensions: @@ -64,7 +65,8 @@ class BaseRenderWorker(Base): self.args = args or {} self.date_created = datetime.now() self.renderer = self.engine.name() - self.renderer_version = self.engine().version() + self.renderer_path = engine_path + self.renderer_version = self.engine(engine_path).version() self.custom_renderer_path = None self.priority = priority self.parent = parent @@ -159,7 +161,7 @@ class BaseRenderWorker(Base): self.errors.append(msg) return - if not self.engine.default_renderer_path() and not self.custom_renderer_path: + if not os.path.exists(self.renderer_path): self.status = RenderStatus.ERROR msg = 'Cannot find render engine path for {}'.format(self.engine.name()) logger.error(msg) @@ -168,7 +170,7 @@ class BaseRenderWorker(Base): self.status = RenderStatus.RUNNING self.start_time = datetime.now() - logger.info(f'Starting {self.engine.name()} {self.engine().version()} Render for {self.input_path} | ' + logger.info(f'Starting {self.engine.name()} {self.renderer_version} Render for {self.input_path} | ' f'Frame Count: {self.total_frames}') self.__thread.start() @@ -183,7 +185,7 @@ class BaseRenderWorker(Base): with open(self.log_path(), "a") as f: - f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine().version()} " + f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.renderer_version} " f"render for {self.input_path}\n\n") f.write(f"Running command: {subprocess_cmds}\n") f.write('=' * 80 + '\n\n') diff --git a/src/workers/blender_worker.py b/src/workers/blender_worker.py index 20fa943..970146d 100644 --- a/src/workers/blender_worker.py +++ b/src/workers/blender_worker.py @@ -11,9 +11,9 @@ class BlenderRenderWorker(BaseRenderWorker): engine = Blender - def __init__(self, input_path, output_path, args=None, parent=None, name=None): - super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args, - parent=parent, name=name) + def __init__(self, input_path, output_path, engine_path, args=None, parent=None, name=None): + super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path, + engine_path=engine_path, args=args, parent=parent, name=name) # Args self.blender_engine = self.args.get('engine', 'BLENDER_EEVEE').upper() @@ -24,7 +24,7 @@ class BlenderRenderWorker(BaseRenderWorker): self.__frame_percent_complete = 0.0 # Scene Info - self.scene_info = Blender().get_scene_info(input_path) + self.scene_info = Blender(engine_path).get_scene_info(input_path) self.start_frame = int(self.scene_info.get('start_frame', 1)) self.end_frame = int(self.scene_info.get('end_frame', self.start_frame)) self.project_length = (self.end_frame - self.start_frame) + 1 @@ -32,7 +32,7 @@ class BlenderRenderWorker(BaseRenderWorker): def generate_worker_subprocess(self): - cmd = [self.engine.default_renderer_path()] + cmd = [self.renderer_path] if self.args.get('background', True): # optionally run render not in background cmd.append('-b') cmd.append(self.input_path) diff --git a/src/workers/ffmpeg_worker.py b/src/workers/ffmpeg_worker.py index 218ce83..81fe5e7 100644 --- a/src/workers/ffmpeg_worker.py +++ b/src/workers/ffmpeg_worker.py @@ -14,7 +14,7 @@ class FFMPEGRenderWorker(BaseRenderWorker): super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name) - stream_info = subprocess.check_output([self.engine.default_renderer_path(), "-i", # https://stackoverflow.com/a/61604105 + stream_info = subprocess.check_output([self.renderer_path, "-i", # https://stackoverflow.com/a/61604105 input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y", "/dev/null"], stderr=subprocess.STDOUT).decode('utf-8') found_frames = re.findall('frame=\s*(\d+)', stream_info) diff --git a/src/workers/worker_factory.py b/src/workers/worker_factory.py index fc14b25..5497629 100644 --- a/src/workers/worker_factory.py +++ b/src/workers/worker_factory.py @@ -1,3 +1,9 @@ +import logging +from ..engines.engine_manager import EngineManager + +logger = logging.getLogger() + + class RenderWorkerFactory: @staticmethod @@ -10,9 +16,26 @@ class RenderWorkerFactory: return classes @staticmethod - def create_worker(renderer, input_path, output_path, args=None, parent=None, name=None): + def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None): + worker_class = RenderWorkerFactory.class_for_name(renderer) - return worker_class(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name) + + # find correct engine version + all_versions = EngineManager.all_versions_for_engine(renderer) + if not all_versions: + raise FileNotFoundError(f"Cannot find any installed {renderer} engines") + + engine_path = all_versions[0]['path'] + if engine_version: + for ver in all_versions: + if ver['version'] == engine_version: + engine_path = ver['path'] + break + if not engine_path: + logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}") + + return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args, + parent=parent, name=name) @staticmethod def supported_renderers():