Download frames from subjobs as frames are completed (#87)

* Add a frame complete notification to BaseWorker and distributed_job_manager.py

* Add API to download individual files to API server and ServerProxy

* Rename subjob notification API and add download_missing_frames_from_subjob

* Subjobs will now notify parent when a frame is complete

* Fix missed rename

* Add some misc logging

* Better error handling

* Fix frame download file path issue

* Download missing frames at job completion and misc cleanup

* Misc cleanup

* Code cleanup

src/api/api_server.py

@@ -134,12 +134,12 @@ def filtered_jobs_json(status_val):
         return f'Cannot find jobs with status {status_val}', 400
 
 
-@server.post('/api/job/<job_id>/notify_parent_of_status_change')
-def subjob_status_change(job_id):
+@server.post('/api/job/<job_id>/send_subjob_update_notification')
+def subjob_update_notification(job_id):
     try:
         subjob_details = request.json
         logger.info(f"Subjob to job id: {job_id} is now {subjob_details['status']}")
-        DistributedJobManager.handle_subjob_status_change(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
+        DistributedJobManager.handle_subjob_update_notification(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
         return Response(status=200)
     except JobNotFoundError:
         return "Job not found", 404
@@ -168,7 +168,22 @@ def get_job_logs(job_id):
 
 @server.get('/api/job/<job_id>/file_list')
 def get_file_list(job_id):
-    return RenderQueue.job_with_id(job_id).file_list()
+    return [os.path.basename(x) for x in RenderQueue.job_with_id(job_id).file_list()]
+
+
+@server.route('/api/job/<job_id>/download')
+def download_file(job_id):
+
+    requested_filename = request.args.get('filename')
+    if not requested_filename:
+        return 'Filename required', 400
+
+    found_job = RenderQueue.job_with_id(job_id)
+    for job_filename in found_job.file_list():
+        if os.path.basename(job_filename).lower() == requested_filename.lower():
+            return send_file(job_filename, as_attachment=True, )
+
+    return f"File '{requested_filename}' not found", 404
 
 
 @server.route('/api/job/<job_id>/download_all')
@@ -293,7 +308,10 @@ def delete_job(job_id):
         if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
             shutil.rmtree(output_dir)
 
-        PreviewManager.delete_previews_for_job(found_job)
+        try:
+            PreviewManager.delete_previews_for_job(found_job)
+        except Exception as e:
+            logger.error(f"Error deleting previews for {found_job}: {e}")
 
         # See if we own the project_dir (i.e. was it uploaded)
         project_dir = os.path.dirname(os.path.dirname(found_job.input_path))
@@ -473,7 +491,10 @@ def get_disk_benchmark():
 def start_server():
     def eval_loop(delay_sec=1):
         while True:
-            RenderQueue.evaluate_queue()
+            try:
+                RenderQueue.evaluate_queue()
+            except Exception as e:
+                logger.error(f"Uncaught error while evaluating queue: {e}")
             time.sleep(delay_sec)
 
     try:

src/api/preview_manager.py

@@ -33,6 +33,7 @@ class PreviewManager:
             logger.warning(f"No valid image or video files found in files from job: {job}")
             return
 
+        os.makedirs(cls.storage_path, exist_ok=True)
         base_path = os.path.join(cls.storage_path, f"{job.id}-{preview_label}-{max_width}")
         preview_video_path = base_path + '.mp4'
         preview_image_path = base_path + '.jpg'
@@ -80,21 +81,24 @@ class PreviewManager:
     @classmethod
     def get_previews_for_job(cls, job):
 
-        directory_path = Path(cls.storage_path)
-        preview_files_for_job = [f for f in directory_path.iterdir() if f.is_file() and f.name.startswith(job.id)]
-
         results = {}
-        for preview_filename in preview_files_for_job:
-            try:
-                pixel_width = str(preview_filename).split('-')[-1]
-                preview_label = str(os.path.basename(preview_filename)).split('-')[1]
-                extension = os.path.splitext(preview_filename)[-1].lower()
-                kind = 'video' if extension in supported_video_formats else \
-                    'image' if extension in supported_image_formats else 'unknown'
-                results[preview_label] = results.get(preview_label, [])
-                results[preview_label].append({'filename': str(preview_filename), 'width': pixel_width, 'kind': kind})
-            except IndexError:  # ignore invalid filenames
-                pass
+        try:
+            directory_path = Path(cls.storage_path)
+            preview_files_for_job = [f for f in directory_path.iterdir() if f.is_file() and f.name.startswith(job.id)]
+
+            for preview_filename in preview_files_for_job:
+                try:
+                    pixel_width = str(preview_filename).split('-')[-1]
+                    preview_label = str(os.path.basename(preview_filename)).split('-')[1]
+                    extension = os.path.splitext(preview_filename)[-1].lower()
+                    kind = 'video' if extension in supported_video_formats else \
+                        'image' if extension in supported_image_formats else 'unknown'
+                    results[preview_label] = results.get(preview_label, [])
+                    results[preview_label].append({'filename': str(preview_filename), 'width': pixel_width, 'kind': kind})
+                except IndexError:  # ignore invalid filenames
+                    pass
+        except FileNotFoundError:
+            pass
         return results
 
     @classmethod

src/api/server_proxy.py

@@ -171,19 +171,19 @@ class RenderServerProxy:
     def get_all_engines(self):
         return self.request_data('all_engines')
 
-    def notify_parent_of_status_change(self, parent_id, subjob):
+    def send_subjob_update_notification(self, parent_id, subjob):
         """
-        Notifies the parent job of a status change in a subjob.
+        Notifies the parent job of an update in a subjob.
 
         Args:
             parent_id (str): The ID of the parent job.
-            subjob (Job): The subjob that has changed status.
+            subjob (Job): The subjob that has updated.
 
         Returns:
             Response: The response from the server.
         """
         hostname = LOOPBACK if self.is_localhost else self.hostname
-        return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/notify_parent_of_status_change',
+        return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
                              json=subjob.json())
 
     def post_job_to_server(self, file_path, job_list, callback=None):
@@ -232,19 +232,27 @@ class RenderServerProxy:
         except Exception as e:
             logger.error(f"An error occurred: {e}")
 
-    def get_job_files(self, job_id, save_path):
+    def get_job_files_list(self, job_id):
+        return self.request_data(f"job/{job_id}/file_list")
+
+    def download_all_job_files(self, job_id, save_path):
         hostname = LOOPBACK if self.is_localhost else self.hostname
         url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
-        return self.download_file(url, filename=save_path)
+        return self.__download_file_from_url(url, output_filepath=save_path)
+
+    def download_job_file(self, job_id, job_filename, save_path):
+        hostname = LOOPBACK if self.is_localhost else self.hostname
+        url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
+        return self.__download_file_from_url(url, output_filepath=save_path)
 
     @staticmethod
-    def download_file(url, filename):
+    def __download_file_from_url(url, output_filepath):
         with requests.get(url, stream=True) as r:
             r.raise_for_status()
-            with open(filename, 'wb') as f:
+            with open(output_filepath, 'wb') as f:
                 for chunk in r.iter_content(chunk_size=8192):
                     f.write(chunk)
-        return filename
+        return output_filepath
 
     # --- Renderer --- #