Integrate watchdog into render worker (#88)

* Add a watchdog to base_worker

* Logging cleanup

* Prevent multiple watchdogs from running if render process restarts

* Add process timeout parameter to Config

* Refactor

* Add error handling to process output parsing

* Fix issue where start_time was not getting set consistently
This commit is contained in:
2024-08-06 10:48:24 -05:00
committed by GitHub
parent 90d5e9b7af
commit 6afb6e65a6
5 changed files with 99 additions and 41 deletions

View File

@@ -10,7 +10,7 @@ class Config:
max_content_path = 100000000
server_log_level = 'debug'
log_buffer_length = 250
subjob_connection_timeout = 120
worker_process_timeout = 120
flask_log_level = 'error'
flask_debug_enable = False
queue_eval_seconds = 1
@@ -28,7 +28,7 @@ class Config:
cls.max_content_path = cfg.get('max_content_path', cls.max_content_path)
cls.server_log_level = cfg.get('server_log_level', cls.server_log_level)
cls.log_buffer_length = cfg.get('log_buffer_length', cls.log_buffer_length)
cls.subjob_connection_timeout = cfg.get('subjob_connection_timeout', cls.subjob_connection_timeout)
cls.worker_process_timeout = cfg.get('worker_process_timeout', cls.worker_process_timeout)
cls.flask_log_level = cfg.get('flask_log_level', cls.flask_log_level)
cls.flask_debug_enable = cfg.get('flask_debug_enable', cls.flask_debug_enable)
cls.queue_eval_seconds = cfg.get('queue_eval_seconds', cls.queue_eval_seconds)