diff --git a/src/distributed_job_manager.py b/src/distributed_job_manager.py index c6300be..b5e9f3a 100644 --- a/src/distributed_job_manager.py +++ b/src/distributed_job_manager.py @@ -143,7 +143,7 @@ class DistributedJobManager: return {k: v for k, v in local_job.children.items() if 'download_status' not in v or v['download_status'] == 'working' or v['download_status'] is None} - logger.debug(f'subjobs_not_downloaded: {subjobs_not_downloaded()}') + logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {local_job.id}') while len(subjobs_not_downloaded()): for child_key, subjob_cached_data in subjobs_not_downloaded().items(): diff --git a/src/workers/base_worker.py b/src/workers/base_worker.py index 84b55a5..c6ccc83 100644 --- a/src/workers/base_worker.py +++ b/src/workers/base_worker.py @@ -176,43 +176,60 @@ class BaseRenderWorker(Base): log_dir = os.path.dirname(self.log_path()) os.makedirs(log_dir, exist_ok=True) - # Start process and get updates subprocess_cmds = self.generate_subprocess() - logger.debug("Renderer commands generated - {}".format(" ".join(subprocess_cmds))) - self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=False) + initial_file_count = len(self.file_list()) + attempt_number = 0 with open(self.log_path(), "a") as f: f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine.version()} " - f"Render for {self.input_path}") - f.write(f"Running command: {' '.join(subprocess_cmds)}\n") - for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding - f.write(c) - # logger.debug(f"{self.engine.name()}Worker: {c.strip()}") - self.last_output = c.strip() - self._parse_stdout(c.strip()) - f.write('\n') + f"render for {self.input_path}\n\n") + f.write(f"Running command: {subprocess_cmds}\n") + f.write('=' * 80 + '\n\n') - # Check return codes - return_code = self.__process.wait() - self.end_time = datetime.now() - # Return early if job was cancelled - if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]: - return + while True: + # Log attempt # + if attempt_number: + f.write(f'\n{"=" * 80} Attempt #{attempt_number} {"=" * 30}\n\n') + logger.warning(f"Restarting render - Attempt #{attempt_number}") + attempt_number += 1 - if return_code: - message = f"{self.engine.name()} render failed with code {return_code} after {self.time_elapsed()}" - logger.error(message) + # Start process and get updates + self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=False) + + for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding + f.write(c) + self.last_output = c.strip() + self._parse_stdout(c.strip()) + + f.write('\n') + + # Check return codes and process + return_code = self.__process.wait() + self.end_time = datetime.now() + + if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]: # user cancelled + message = f"{self.engine.name()} render ended with status '{self.status}' " \ + f"after {self.time_elapsed()}" + f.write(message) + return + + if not return_code: + message = f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in {self.time_elapsed()}" + f.write(message) + break + + # Handle non-zero return codes + message = f"{'=' * 50}\n\n{self.engine.name()} render failed with code {return_code} " \ + f"after {self.time_elapsed()}" f.write(message) - self.status = RenderStatus.ERROR - if not self.errors: - self.errors = [message] - return + self.errors.append(message) - message = f"{self.engine.name()} render completed successfully in {self.time_elapsed()}" - logger.info(message) - f.write(message) + # if file output hasn't increased, return as error, otherwise restart process. + if len(self.file_list()) <= initial_file_count: + self.status = RenderStatus.ERROR + return if self.children: from src.distributed_job_manager import DistributedJobManager diff --git a/src/workers/blender_worker.py b/src/workers/blender_worker.py index c45fa11..72ac829 100644 --- a/src/workers/blender_worker.py +++ b/src/workers/blender_worker.py @@ -37,8 +37,13 @@ class BlenderRenderWorker(BaseRenderWorker): cmd.append('-b') cmd.append(self.input_path) + # Python expressions + cmd.append('--python-expr') + python_exp = 'import bpy; bpy.context.scene.render.use_overwrite = False;' if self.camera: - cmd.extend(['--python-expr', f"import bpy;bpy.context.scene.camera = bpy.data.objects['{self.camera}'];"]) + python_exp = python_exp + f"bpy.context.scene.camera = bpy.data.objects['{self.camera}'];" + # insert any other python exp checks here + cmd.append(python_exp) path_without_ext = os.path.splitext(self.output_path)[0] + "_" cmd.extend(['-E', self.blender_engine, '-o', path_without_ext, '-F', self.export_format]) diff --git a/start_client.py b/start_client.py old mode 100644 new mode 100755