Fix #20 - Relaunch renderer when renderer crashes (#28)

* Render worker makes multiple attempts if file_path increases on each run

* Change blender_worker.py to not overwrite existing files

* Chmod +x to client

src/distributed_job_manager.py

@@ -143,7 +143,7 @@ class DistributedJobManager:
             return {k: v for k, v in local_job.children.items() if 'download_status' not in v or
                     v['download_status'] == 'working' or v['download_status'] is None}
 
-        logger.debug(f'subjobs_not_downloaded: {subjobs_not_downloaded()}')
+        logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {local_job.id}')
 
         while len(subjobs_not_downloaded()):
             for child_key, subjob_cached_data in subjobs_not_downloaded().items():

src/workers/base_worker.py

@@ -176,43 +176,60 @@ class BaseRenderWorker(Base):
         log_dir = os.path.dirname(self.log_path())
         os.makedirs(log_dir, exist_ok=True)
 
-        # Start process and get updates
         subprocess_cmds = self.generate_subprocess()
-        logger.debug("Renderer commands generated - {}".format(" ".join(subprocess_cmds)))
-        self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                                          universal_newlines=False)
+        initial_file_count = len(self.file_list())
+        attempt_number = 0
 
         with open(self.log_path(), "a") as f:
 
             f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.engine.version()} "
-                    f"Render for {self.input_path}")
-            f.write(f"Running command: {' '.join(subprocess_cmds)}\n")
-            for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"):  # or another encoding
-                f.write(c)
-                # logger.debug(f"{self.engine.name()}Worker: {c.strip()}")
-                self.last_output = c.strip()
-                self._parse_stdout(c.strip())
-            f.write('\n')
+                    f"render for {self.input_path}\n\n")
+            f.write(f"Running command: {subprocess_cmds}\n")
+            f.write('=' * 80 + '\n\n')
 
-            # Check return codes
-            return_code = self.__process.wait()
-            self.end_time = datetime.now()
-            # Return early if job was cancelled
-            if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
-                return
+            while True:
+                # Log attempt #
+                if attempt_number:
+                    f.write(f'\n{"=" * 80} Attempt #{attempt_number} {"=" * 30}\n\n')
+                    logger.warning(f"Restarting render - Attempt #{attempt_number}")
+                attempt_number += 1
 
-            if return_code:
-                message = f"{self.engine.name()} render failed with code {return_code} after {self.time_elapsed()}"
-                logger.error(message)
+                # Start process and get updates
+                self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                                  universal_newlines=False)
+
+                for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"):  # or another encoding
+                    f.write(c)
+                    self.last_output = c.strip()
+                    self._parse_stdout(c.strip())
+
+                f.write('\n')
+
+                # Check return codes and process
+                return_code = self.__process.wait()
+                self.end_time = datetime.now()
+
+                if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:  # user cancelled
+                    message = f"{self.engine.name()} render ended with status '{self.status}' " \
+                              f"after {self.time_elapsed()}"
+                    f.write(message)
+                    return
+
+                if not return_code:
+                    message = f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in {self.time_elapsed()}"
+                    f.write(message)
+                    break
+
+                # Handle non-zero return codes
+                message = f"{'=' * 50}\n\n{self.engine.name()} render failed with code {return_code} " \
+                          f"after {self.time_elapsed()}"
                 f.write(message)
-                self.status = RenderStatus.ERROR
-                if not self.errors:
-                    self.errors = [message]
-                return
+                self.errors.append(message)
 
-            message = f"{self.engine.name()} render completed successfully in {self.time_elapsed()}"
-            logger.info(message)
-            f.write(message)
+                # if file output hasn't increased, return as error, otherwise restart process.
+                if len(self.file_list()) <= initial_file_count:
+                    self.status = RenderStatus.ERROR
+                    return
 
         if self.children:
             from src.distributed_job_manager import DistributedJobManager

src/workers/blender_worker.py

@@ -37,8 +37,13 @@ class BlenderRenderWorker(BaseRenderWorker):
             cmd.append('-b')
         cmd.append(self.input_path)
 
+        # Python expressions
+        cmd.append('--python-expr')
+        python_exp = 'import bpy; bpy.context.scene.render.use_overwrite = False;'
         if self.camera:
-            cmd.extend(['--python-expr', f"import bpy;bpy.context.scene.camera = bpy.data.objects['{self.camera}'];"])
+            python_exp = python_exp + f"bpy.context.scene.camera = bpy.data.objects['{self.camera}'];"
+        # insert any other python exp checks here
+        cmd.append(python_exp)
 
         path_without_ext = os.path.splitext(self.output_path)[0] + "_"
         cmd.extend(['-E', self.blender_engine, '-o', path_without_ext, '-F', self.export_format])