From 1bbf11a9381b72f96ca7e98abae072a80cd03b91 Mon Sep 17 00:00:00 2001 From: Brett Date: Sat, 21 Oct 2023 22:04:37 -0700 Subject: [PATCH] Fix path lookups and add engine_path to workers --- src/api_server.py | 3 ++- src/engines/base_engine.py | 2 ++ src/engines/blender_engine.py | 17 +++++++++-------- src/engines/ffmpeg_engine.py | 2 +- src/render_queue.py | 5 +++-- src/workers/base_worker.py | 12 +++++++----- src/workers/blender_worker.py | 10 +++++----- src/workers/ffmpeg_worker.py | 2 +- src/workers/worker_factory.py | 27 +++++++++++++++++++++++++-- 9 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/api_server.py b/src/api_server.py index ff05c12..47ccf6a 100755 --- a/src/api_server.py +++ b/src/api_server.py @@ -391,7 +391,7 @@ def add_job_handler(): try: # prepare output paths output_dir = os.path.join(job_dir, job_data.get('name') if len(jobs_list) > 1 else 'output') - os.makedirs(output_dir, exist_ok=True) + os.makedirs(system_safe_path(output_dir), exist_ok=True) # get new output path in output_dir job_data['output_path'] = os.path.join(output_dir, os.path.basename( @@ -402,6 +402,7 @@ def add_job_handler(): worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'], input_path=loaded_project_local_path, output_path=job_data["output_path"], + engine_version=job_data.get('engine_version'), args=job_data.get('args', {})) worker.status = job_data.get("initial_status", worker.status) worker.parent = job_data.get("parent", worker.parent) diff --git a/src/engines/base_engine.py b/src/engines/base_engine.py index 1e668a8..ccb3b07 100644 --- a/src/engines/base_engine.py +++ b/src/engines/base_engine.py @@ -13,6 +13,8 @@ class BaseRenderEngine(object): def __init__(self, custom_path=None): self.custom_renderer_path = custom_path + if not self.renderer_path(): + raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance") def renderer_path(self): return self.custom_renderer_path or self.default_renderer_path() diff --git a/src/engines/blender_engine.py b/src/engines/blender_engine.py index 43e1f82..6d3ab39 100644 --- a/src/engines/blender_engine.py +++ b/src/engines/blender_engine.py @@ -37,18 +37,17 @@ class Blender(BaseRenderEngine): return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression], capture_output=True, timeout=timeout) except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") - pass + logger.error(f"Error running python expression in blender: {e}") else: raise FileNotFoundError(f'Project file not found: {project_path}') def run_python_script(self, project_path, script_path, timeout=None): if os.path.exists(project_path) and os.path.exists(script_path): try: - return subprocess.run([self.default_renderer_path(), '-b', project_path, '--python', script_path], + return subprocess.run([self.renderer_path(), '-b', project_path, '--python', script_path], capture_output=True, timeout=timeout) except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") + logger.warning(f"Error running python script in blender: {e}") pass elif not os.path.exists(project_path): raise FileNotFoundError(f'Project file not found: {project_path}') @@ -59,8 +58,9 @@ class Blender(BaseRenderEngine): def get_scene_info(self, project_path, timeout=10): scene_info = {} try: - results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'get_file_info.py'), timeout=timeout) + script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'get_file_info.py') + results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout) result_text = results.stdout.decode() for line in result_text.splitlines(): if line.startswith('SCENE_DATA:'): @@ -77,8 +77,9 @@ class Blender(BaseRenderEngine): # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 try: logger.info(f"Starting to pack Blender file: {project_path}") - results = self.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'blender', 'pack_project.py'), timeout=timeout) + script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'pack_project.py') + results = self.run_python_script(project_path, system_safe_path(script_path), timeout=timeout) result_text = results.stdout.decode() dir_name = os.path.dirname(project_path) diff --git a/src/engines/ffmpeg_engine.py b/src/engines/ffmpeg_engine.py index a665bdd..52e59b5 100644 --- a/src/engines/ffmpeg_engine.py +++ b/src/engines/ffmpeg_engine.py @@ -59,7 +59,7 @@ class FFMPEG(BaseRenderEngine): return [x for x in self.get_all_formats() if 'E' in x['type'].upper()] def get_frame_count(self, path_to_file): - raw_stdout = subprocess.check_output([self.default_renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', + raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy', '-f', 'null', '-'], stderr=subprocess.STDOUT, timeout=SUBPROCESS_TIMEOUT).decode('utf-8') match = re.findall(r'frame=\s*(\d+)', raw_stdout) diff --git a/src/render_queue.py b/src/render_queue.py index f8f3c4a..d8f2bda 100755 --- a/src/render_queue.py +++ b/src/render_queue.py @@ -5,7 +5,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.utilities.status_utils import RenderStatus -from src.workers.worker_factory import RenderWorkerFactory +from src.engines.engine_manager import EngineManager from src.workers.base_worker import Base logger = logging.getLogger() @@ -100,7 +100,8 @@ class RenderQueue: @classmethod def is_available_for_job(cls, renderer, priority=2): - if not RenderWorkerFactory.class_for_name(renderer).engine.default_renderer_path(): + + if not EngineManager.all_versions_for_engine(renderer): return False instances = cls.renderer_instances() diff --git a/src/workers/base_worker.py b/src/workers/base_worker.py index 8f52c58..44eebfa 100644 --- a/src/workers/base_worker.py +++ b/src/workers/base_worker.py @@ -30,6 +30,7 @@ class BaseRenderWorker(Base): end_time = Column(DateTime, nullable=True) renderer = Column(String) renderer_version = Column(String) + renderer_path = Column(String) priority = Column(Integer) project_length = Column(Integer) start_frame = Column(Integer) @@ -42,7 +43,7 @@ class BaseRenderWorker(Base): engine = None - def __init__(self, input_path, output_path, priority=2, args=None, ignore_extensions=True, parent=None, + def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None, name=None): if not ignore_extensions: @@ -64,7 +65,8 @@ class BaseRenderWorker(Base): self.args = args or {} self.date_created = datetime.now() self.renderer = self.engine.name() - self.renderer_version = self.engine().version() + self.renderer_path = engine_path + self.renderer_version = self.engine(engine_path).version() self.custom_renderer_path = None self.priority = priority self.parent = parent @@ -159,7 +161,7 @@ class BaseRenderWorker(Base): self.errors.append(msg) return - if not self.engine.default_renderer_path() and not self.custom_renderer_path: + if not os.path.exists(self.renderer_path): self.status = RenderStatus.ERROR msg = 'Cannot find render engine path for {}'.format(self.engine.name()) logger.error(msg) @@ -168,7 +170,7 @@ class BaseRenderWorker(Base): self.status = RenderStatus.RUNNING self.start_time = datetime.now() - logger.info(f'Starting {self.engine.name()} {self.engine().version()} Render for {self.input_path} | ' + logger.info(f'Starting {self.engine.name()} {self.renderer_version} Render for {self.input_path} | ' f'Frame Count: {self.total_frames}') self.__thread.start() @@ -183,7 +185,7 @@ class BaseRenderWorker(Base): with open(self.log_path(), "a") as f: - f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine().version()} " + f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.renderer_version} " f"render for {self.input_path}\n\n") f.write(f"Running command: {subprocess_cmds}\n") f.write('=' * 80 + '\n\n') diff --git a/src/workers/blender_worker.py b/src/workers/blender_worker.py index 20fa943..970146d 100644 --- a/src/workers/blender_worker.py +++ b/src/workers/blender_worker.py @@ -11,9 +11,9 @@ class BlenderRenderWorker(BaseRenderWorker): engine = Blender - def __init__(self, input_path, output_path, args=None, parent=None, name=None): - super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args, - parent=parent, name=name) + def __init__(self, input_path, output_path, engine_path, args=None, parent=None, name=None): + super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path, + engine_path=engine_path, args=args, parent=parent, name=name) # Args self.blender_engine = self.args.get('engine', 'BLENDER_EEVEE').upper() @@ -24,7 +24,7 @@ class BlenderRenderWorker(BaseRenderWorker): self.__frame_percent_complete = 0.0 # Scene Info - self.scene_info = Blender().get_scene_info(input_path) + self.scene_info = Blender(engine_path).get_scene_info(input_path) self.start_frame = int(self.scene_info.get('start_frame', 1)) self.end_frame = int(self.scene_info.get('end_frame', self.start_frame)) self.project_length = (self.end_frame - self.start_frame) + 1 @@ -32,7 +32,7 @@ class BlenderRenderWorker(BaseRenderWorker): def generate_worker_subprocess(self): - cmd = [self.engine.default_renderer_path()] + cmd = [self.renderer_path] if self.args.get('background', True): # optionally run render not in background cmd.append('-b') cmd.append(self.input_path) diff --git a/src/workers/ffmpeg_worker.py b/src/workers/ffmpeg_worker.py index 218ce83..81fe5e7 100644 --- a/src/workers/ffmpeg_worker.py +++ b/src/workers/ffmpeg_worker.py @@ -14,7 +14,7 @@ class FFMPEGRenderWorker(BaseRenderWorker): super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name) - stream_info = subprocess.check_output([self.engine.default_renderer_path(), "-i", # https://stackoverflow.com/a/61604105 + stream_info = subprocess.check_output([self.renderer_path, "-i", # https://stackoverflow.com/a/61604105 input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y", "/dev/null"], stderr=subprocess.STDOUT).decode('utf-8') found_frames = re.findall('frame=\s*(\d+)', stream_info) diff --git a/src/workers/worker_factory.py b/src/workers/worker_factory.py index fc14b25..5497629 100644 --- a/src/workers/worker_factory.py +++ b/src/workers/worker_factory.py @@ -1,3 +1,9 @@ +import logging +from ..engines.engine_manager import EngineManager + +logger = logging.getLogger() + + class RenderWorkerFactory: @staticmethod @@ -10,9 +16,26 @@ class RenderWorkerFactory: return classes @staticmethod - def create_worker(renderer, input_path, output_path, args=None, parent=None, name=None): + def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None): + worker_class = RenderWorkerFactory.class_for_name(renderer) - return worker_class(input_path=input_path, output_path=output_path, args=args, parent=parent, name=name) + + # find correct engine version + all_versions = EngineManager.all_versions_for_engine(renderer) + if not all_versions: + raise FileNotFoundError(f"Cannot find any installed {renderer} engines") + + engine_path = all_versions[0]['path'] + if engine_version: + for ver in all_versions: + if ver['version'] == engine_version: + engine_path = ver['path'] + break + if not engine_path: + logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}") + + return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args, + parent=parent, name=name) @staticmethod def supported_renderers():