From 220b3fcc25afca7e86a196561bfcdff32ced0cf6 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Sat, 3 Aug 2024 20:55:22 -0500 Subject: [PATCH] Streamline job runtime - improve logging --- src/engines/core/base_worker.py | 42 +++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/engines/core/base_worker.py b/src/engines/core/base_worker.py index a77b696..5ac6d92 100644 --- a/src/engines/core/base_worker.py +++ b/src/engines/core/base_worker.py @@ -79,9 +79,9 @@ class BaseRenderWorker(Base): self.maximum_attempts = 3 # Frame Ranges - self.project_length = -1 - self.current_frame = 0 # should this be a 1 ? - self.start_frame = 0 # should this be a 1 ? + self.project_length = 0 # is this necessary? + self.current_frame = 0 + self.start_frame = 0 self.end_frame = None # Logging @@ -186,7 +186,7 @@ class BaseRenderWorker(Base): subprocess_cmds = self.generate_subprocess() initial_file_count = len(self.file_list()) - attempt_number = 0 + failed_attempts = 0 with open(self.log_path(), "a") as f: @@ -197,17 +197,17 @@ class BaseRenderWorker(Base): while True: # Log attempt # - if attempt_number: - if attempt_number >= self.maximum_attempts: - err_msg = f"Maximum attempts exceeded ({attempt_number})" + if failed_attempts: + if failed_attempts >= self.maximum_attempts: + err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})" logger.error(err_msg) self.status = RenderStatus.ERROR self.errors.append(err_msg) return else: - f.write(f'\n{"=" * 80} Attempt #{attempt_number} {"=" * 30}\n\n') - logger.warning(f"Restarting render - Attempt #{attempt_number}") - attempt_number += 1 + f.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n') + logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}") + self.status = RenderStatus.RUNNING # Start process and get updates self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -215,6 +215,8 @@ class BaseRenderWorker(Base): for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding f.write(c) + f.flush() + os.fsync(f.fileno()) self.last_output = c.strip() self._parse_stdout(c.strip()) @@ -230,22 +232,26 @@ class BaseRenderWorker(Base): f.write(message) return - if not return_code: + # if file output hasn't increased, return as error, otherwise restart process. + if len(self.file_list()) <= initial_file_count: + err_msg = f"File count has not increased. Count is still {len(self.file_list())}" + f.write(f'Error: {err_msg}\n\n') + self.errors.append(err_msg) + self.status = RenderStatus.ERROR + + # Handle completed - All else counts as failed attempt + if (self.status == RenderStatus.COMPLETED) and not return_code: message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in " - f"{self.time_elapsed()}") + f"{self.time_elapsed()}\n") f.write(message) break # Handle non-zero return codes message = f"{'=' * 50}\n\n{self.engine.name()} render failed with code {return_code} " \ - f"after {self.time_elapsed()}" + f"after {self.time_elapsed()}\n\n" f.write(message) self.errors.append(message) - - # if file output hasn't increased, return as error, otherwise restart process. - if len(self.file_list()) <= initial_file_count: - self.status = RenderStatus.ERROR - return + failed_attempts += 1 if self.children: from src.distributed_job_manager import DistributedJobManager