Streamline job runtime - improve logging

src/engines/core/base_worker.py

@@ -79,9 +79,9 @@ class BaseRenderWorker(Base):
         self.maximum_attempts = 3
 
         # Frame Ranges
-        self.project_length = -1
+        self.project_length = 0  # is this necessary?
-        self.current_frame = 0  # should this be a 1 ?
+        self.current_frame = 0
-        self.start_frame = 0  # should this be a 1 ?
+        self.start_frame = 0
         self.end_frame = None
 
         # Logging
@@ -186,7 +186,7 @@ class BaseRenderWorker(Base):
 
         subprocess_cmds = self.generate_subprocess()
         initial_file_count = len(self.file_list())
-        attempt_number = 0
+        failed_attempts = 0
 
         with open(self.log_path(), "a") as f:
 
@@ -197,17 +197,17 @@ class BaseRenderWorker(Base):
 
             while True:
                 # Log attempt #
-                if attempt_number:
+                if failed_attempts:
-                    if attempt_number >= self.maximum_attempts:
+                    if failed_attempts >= self.maximum_attempts:
-                        err_msg = f"Maximum attempts exceeded ({attempt_number})"
+                        err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
                         logger.error(err_msg)
                         self.status = RenderStatus.ERROR
                         self.errors.append(err_msg)
                         return
                     else:
-                        f.write(f'\n{"=" * 80} Attempt #{attempt_number} {"=" * 30}\n\n')
+                        f.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
-                        logger.warning(f"Restarting render - Attempt #{attempt_number}")
+                        logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
-                attempt_number += 1
+                        self.status = RenderStatus.RUNNING
 
                 # Start process and get updates
                 self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
@@ -215,6 +215,8 @@ class BaseRenderWorker(Base):
 
                 for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"):  # or another encoding
                     f.write(c)
+                    f.flush()
+                    os.fsync(f.fileno())
                     self.last_output = c.strip()
                     self._parse_stdout(c.strip())
 
@@ -230,22 +232,26 @@ class BaseRenderWorker(Base):
                     f.write(message)
                     return
 
-                if not return_code:
+                # if file output hasn't increased, return as error, otherwise restart process.
+                if len(self.file_list()) <= initial_file_count:
+                    err_msg = f"File count has not increased. Count is still {len(self.file_list())}"
+                    f.write(f'Error: {err_msg}\n\n')
+                    self.errors.append(err_msg)
+                    self.status = RenderStatus.ERROR
+
+                # Handle completed - All else counts as failed attempt
+                if (self.status == RenderStatus.COMPLETED) and not return_code:
                     message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
-                               f"{self.time_elapsed()}")
+                               f"{self.time_elapsed()}\n")
                     f.write(message)
                     break
 
                 # Handle non-zero return codes
                 message = f"{'=' * 50}\n\n{self.engine.name()} render failed with code {return_code} " \
-                          f"after {self.time_elapsed()}"
+                          f"after {self.time_elapsed()}\n\n"
                 f.write(message)
                 self.errors.append(message)
-
+                failed_attempts += 1
-                # if file output hasn't increased, return as error, otherwise restart process.
-                if len(self.file_list()) <= initial_file_count:
-                    self.status = RenderStatus.ERROR
-                    return
 
         if self.children:
             from src.distributed_job_manager import DistributedJobManager