brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2025-12-17 08:48:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Streamline job runtime - improve logging

Browse Source
This commit is contained in:
Brett Williams
2024-08-03 20:55:22 -05:00
parent 82613c3963
commit 220b3fcc25
1 changed files with 24 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/engines/core/base_worker.py
View File

@@ -79,9 +79,9 @@ class BaseRenderWorker(Base):
self.maximum_attempts = 3
# Frame Ranges
self.project_length = -1
self.current_frame = 0 # should this be a 1 ?
self.start_frame = 0 # should this be a 1 ?
self.project_length = 0 # is this necessary?
self.current_frame = 0
self.start_frame = 0
self.end_frame = None
# Logging
@@ -186,7 +186,7 @@ class BaseRenderWorker(Base):
subprocess_cmds = self.generate_subprocess()
initial_file_count = len(self.file_list())
attempt_number = 0
failed_attempts = 0
with open(self.log_path(), "a") as f:
@@ -197,17 +197,17 @@ class BaseRenderWorker(Base):
while True:
# Log attempt #
if attempt_number:
if attempt_number >= self.maximum_attempts:
err_msg = f"Maximum attempts exceeded ({attempt_number})"
if failed_attempts:
if failed_attempts >= self.maximum_attempts:
err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
logger.error(err_msg)
self.status = RenderStatus.ERROR
self.errors.append(err_msg)
return
else:
f.write(f'\n{"=" * 80} Attempt #{attempt_number} {"=" * 30}\n\n')
logger.warning(f"Restarting render - Attempt #{attempt_number}")
attempt_number += 1
f.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
self.status = RenderStatus.RUNNING
# Start process and get updates
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
@@ -215,6 +215,8 @@ class BaseRenderWorker(Base):
for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding
f.write(c)
f.flush()
os.fsync(f.fileno())
self.last_output = c.strip()
self._parse_stdout(c.strip())
@@ -230,22 +232,26 @@ class BaseRenderWorker(Base):
f.write(message)
return
if not return_code:
# if file output hasn't increased, return as error, otherwise restart process.
if len(self.file_list()) <= initial_file_count:
err_msg = f"File count has not increased. Count is still {len(self.file_list())}"
f.write(f'Error: {err_msg}\n\n')
self.errors.append(err_msg)
self.status = RenderStatus.ERROR
# Handle completed - All else counts as failed attempt
if (self.status == RenderStatus.COMPLETED) and not return_code:
message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
f"{self.time_elapsed()}")
f"{self.time_elapsed()}\n")
f.write(message)
break
# Handle non-zero return codes
message = f"{'=' * 50}\n\n{self.engine.name()} render failed with code {return_code} " \
f"after {self.time_elapsed()}"
f"after {self.time_elapsed()}\n\n"
f.write(message)
self.errors.append(message)
# if file output hasn't increased, return as error, otherwise restart process.
if len(self.file_list()) <= initial_file_count:
self.status = RenderStatus.ERROR
return
failed_attempts += 1
if self.children:
from src.distributed_job_manager import DistributedJobManager