Fix issue where jobs were not always deleted (#137)

* Better job deletion logic

* Cleanup UI after deleting runs

src/api/api_server.py

@@ -322,19 +322,11 @@ def delete_job(job_id):
 
         RenderQueue.delete_job(found_job)
 
-        # Delete output directory if we own it
-        if output_dir.exists() and output_dir.is_relative_to(upload_root):
-            shutil.rmtree(output_dir)
-
-        # Delete project directory if we own it and it's unused
-        try:
-            if project_dir.exists() and project_dir.is_relative_to(upload_root):
-                project_dir_files = [p for p in project_dir.iterdir() if not p.name.startswith(".")]
-                if not project_dir_files or (len(project_dir_files) == 1 and "source" in project_dir_files[0].name):
-                    logger.info(f"Removing project directory: {project_dir}")
-                    shutil.rmtree(project_dir)
-        except Exception as e:
-            logger.error(f"Error removing project files: {e}")
+        if _has_remaining_jobs_for_project(project_dir):
+            _delete_job_output(output_path, project_dir, upload_root)
+        elif project_dir.exists() and project_dir.is_relative_to(upload_root):
+            logger.info(f"Removing project directory: {project_dir}")
+            shutil.rmtree(project_dir)
 
         return "Job deleted", 200
 
@@ -343,6 +335,41 @@ def delete_job(job_id):
         return f"Error deleting job: {e}", 500
 
 
+def _has_remaining_jobs_for_project(project_dir):
+    for job in RenderQueue.all_jobs():
+        try:
+            if Path(job.input_path).parent.parent == project_dir:
+                return True
+        except (AttributeError, TypeError):
+            continue
+    return False
+
+
+def _delete_job_output(output_path, project_dir, upload_root):
+    output_dir = output_path.parent
+    project_output_dir = project_dir / 'output'
+
+    if not output_dir.exists() or not output_dir.is_relative_to(upload_root):
+        return
+
+    if output_dir != project_output_dir:
+        shutil.rmtree(output_dir)
+        return
+
+    output_prefix = output_path.stem
+    for output_file in output_dir.iterdir():
+        if _output_file_matches_prefix(output_file.name, output_prefix) and output_file.is_file():
+            output_file.unlink()
+
+
+def _output_file_matches_prefix(filename, output_prefix):
+    return (
+        filename == output_prefix or
+        filename.startswith(f'{output_prefix}_') or
+        filename.startswith(f'{output_prefix}.')
+    )
+
+
 # --------------------------------------------
 # Engine Info and Management:
 # --------------------------------------------

src/api/server_proxy.py

@@ -22,6 +22,8 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat
 
 logger = logging.getLogger()
 OFFLINE_MAX = 4
+JOB_UPLOAD_TIMEOUT = (10, 1800)
+FILE_DOWNLOAD_TIMEOUT = (10, 1800)
 
 
 class RenderServerProxy:
@@ -161,7 +163,7 @@ class RenderServerProxy:
     def get_jobs(self, timeout=5, ignore_token=False):
         if not self.__update_in_background or ignore_token:
             self.__update_job_cache(timeout, ignore_token)
-        return self.__jobs_cache.copy() if self.__jobs_cache else None
+        return self.__jobs_cache.copy()
 
     def get_status(self):
         status = self.request_data('status')
@@ -187,7 +189,7 @@ class RenderServerProxy:
     # Job Lifecycle:
     # --------------------------------------------
 
-    def create_job(self, file_path: Path, job_data, callback=None):
+    def create_job(self, file_path: Path, job_data, callback=None, timeout=JOB_UPLOAD_TIMEOUT):
         """
         Posts a job to the server.
 
@@ -195,6 +197,8 @@ class RenderServerProxy:
             file_path (Path): The path to the file to upload.
             job_data (dict): A dict of jobs data.
             callback (function, optional): A callback function to call during the upload. Defaults to None.
+            timeout (float | tuple, optional): Requests timeout. Defaults to a 10-second connect timeout and
+                30-minute read timeout for large uploads.
 
         Returns:
             Response: The response from the server.
@@ -208,7 +212,7 @@ class RenderServerProxy:
             job_data['local_path'] = str(file_path)
             url = urljoin(f'http://{self.hostname}:{self.port}', '/api/jobs')
             headers = {'Content-Type': 'application/json'}
-            return requests.post(url, data=json.dumps(job_data), headers=headers)
+            return requests.post(url, data=json.dumps(job_data), headers=headers, timeout=timeout)
 
         # Prepare the form data for remote host
         with open(file_path, 'rb') as file:
@@ -223,16 +227,22 @@ class RenderServerProxy:
             url = urljoin(f'http://{self.hostname}:{self.port}', '/api/jobs')
 
             # Send the request with proper resource management
-            with requests.post(url, data=monitor, headers=headers) as response:
+            with requests.post(url, data=monitor, headers=headers, timeout=timeout) as response:
                 return response
 
     def cancel_job(self, job_id, confirm=False):
-        return self._post(f'jobs/{job_id}/cancel', params={'confirm': confirm})
+        response = self._post(f'jobs/{job_id}/cancel', params={'confirm': confirm})
+        if response.ok:
+            self.__update_job_cache(timeout=5, ignore_token=True)
+        return response
 
     def delete_job(self, job_id, confirm=False):
-        return self._post(f'jobs/{job_id}/delete', params={'confirm': confirm})
+        response = self._post(f'jobs/{job_id}/delete', params={'confirm': confirm})
+        if response.ok:
+            self.__update_job_cache(timeout=5, ignore_token=True)
+        return response
 
-    def send_subjob_update_notification(self, parent_id, subjob):
+    def send_subjob_update_notification(self, parent_id, subjob, timeout=5):
         """
         Notifies the parent job of an update in a subjob.
 
@@ -244,7 +254,7 @@ class RenderServerProxy:
             Response: The response from the server.
         """
         return requests.post(f'http://{self.hostname}:{self.port}/api/jobs/{parent_id}/subjob_update',
-                             json=subjob.json())
+                             json=subjob.json(), timeout=timeout)
 
     # --------------------------------------------
     # Engines:
@@ -308,17 +318,17 @@ class RenderServerProxy:
     # Download Files:
     # --------------------------------------------
 
-    def download_all_job_files(self, job_id, save_path):
+    def download_all_job_files(self, job_id, save_path, timeout=FILE_DOWNLOAD_TIMEOUT):
         url = f'http://{self.hostname}:{self.port}/api/jobs/{job_id}/download_all'
-        return self.__download_file_from_url(url, output_filepath=save_path)
+        return self.__download_file_from_url(url, output_filepath=save_path, timeout=timeout)
 
-    def download_job_file(self, job_id, job_filename, save_path):
+    def download_job_file(self, job_id, job_filename, save_path, timeout=FILE_DOWNLOAD_TIMEOUT):
         url = f'http://{self.hostname}:{self.port}/api/jobs/{job_id}/download?filename={job_filename}'
-        return self.__download_file_from_url(url, output_filepath=save_path)
+        return self.__download_file_from_url(url, output_filepath=save_path, timeout=timeout)
 
     @staticmethod
-    def __download_file_from_url(url, output_filepath):
-        with requests.get(url, stream=True) as r:
+    def __download_file_from_url(url, output_filepath, timeout=FILE_DOWNLOAD_TIMEOUT):
+        with requests.get(url, stream=True, timeout=timeout) as r:
             r.raise_for_status()
             with open(output_filepath, 'wb') as f:
                 for chunk in r.iter_content(chunk_size=8192):

src/engines/blender/blender_downloader.py

@@ -70,7 +70,7 @@ class BlenderDownloader(EngineDownloader):
 
     @staticmethod
     def __find_LTS_versions():
-        response = requests.get('https://www.blender.org/download/lts/')
+        response = requests.get('https://www.blender.org/download/lts/', timeout=5)
         response.raise_for_status()
 
         lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/'

src/ui/add_job_window.py

@@ -601,7 +601,7 @@ class SubmitWorker(QThread):
 
             # determine if any cameras are checked
             selected_cameras = []
-            if self.window.cameras_list.count() and not self.window.cameras_group.isHidden():
+            if self.window.cameras_list.count() and self.window.cameras_group.isEnabled():
                 for index in range(self.window.cameras_list.count()):
                     item = self.window.cameras_list.item(index)
                     if item.checkState() == Qt.CheckState.Checked:
@@ -613,7 +613,7 @@ class SubmitWorker(QThread):
                 children_jobs = []
                 for cam in selected_cameras:
                     child_job_data = dict()
-                    child_job_data['args'] = {}
+                    child_job_data['args'] = dict(job_json['args'])
                     child_job_data['args']['camera'] = cam
                     child_job_data['name'] = job_json['name'].replace(' ', '-') + "_" + cam.replace(' ', '')
                     child_job_data['output_path'] = child_job_data['name']

src/ui/engine_help_window.py

@@ -26,5 +26,5 @@ class EngineHelpViewer(QMainWindow):
         self.fetch_help()
 
     def fetch_help(self):
-        result = requests.get(self.help_path)
+        result = requests.get(self.help_path, timeout=10)
         self.text_edit.setPlainText(result.text)

src/ui/log_window.py

@@ -26,5 +26,5 @@ class LogViewer(QMainWindow):
         self.fetch_logs()
 
     def fetch_logs(self):
-        result = requests.get(self.log_path)
+        result = requests.get(self.log_path, timeout=10)
         self.text_edit.setPlainText(result.text)

src/ui/main_window.py

@@ -293,34 +293,36 @@ class MainWindow(QMainWindow):
             return
 
         server_job_data = self.job_data.get(self.current_server_proxy.hostname)
-        if server_job_data:
-            num_jobs = len(server_job_data)
-            self.job_list_view.setRowCount(num_jobs)
+        if server_job_data is None:
+            return
 
-            for row, job in enumerate(server_job_data):
+        num_jobs = len(server_job_data)
+        self.job_list_view.setRowCount(num_jobs)
 
-                display_status = job['status'] if job['status'] != RenderStatus.RUNNING.value else \
-                    ('%.0f%%' % (job['percent_complete'] * 100))  # if running, show percent, otherwise just show status
-                tags = (job['status'],)
-                start_time = datetime.datetime.fromisoformat(job['start_time']) if job['start_time'] else None
-                end_time = datetime.datetime.fromisoformat(job['end_time']) if job['end_time'] else None
+        for row, job in enumerate(server_job_data):
 
-                time_elapsed = "" if (job['status'] != RenderStatus.RUNNING.value and not end_time) else \
-                    get_time_elapsed(start_time, end_time)
+            display_status = job['status'] if job['status'] != RenderStatus.RUNNING.value else \
+                ('%.0f%%' % (job['percent_complete'] * 100))  # if running, show percent, otherwise just show status
+            tags = (job['status'],)
+            start_time = datetime.datetime.fromisoformat(job['start_time']) if job['start_time'] else None
+            end_time = datetime.datetime.fromisoformat(job['end_time']) if job['end_time'] else None
 
-                name = job.get('name') or os.path.basename(job.get('input_path', ''))
-                engine_name = f"{job.get('engine', '')}-{job.get('engine_version')}"
-                priority = str(job.get('priority', ''))
-                total_frames = str(job.get('total_frames', ''))
-                converted_time = datetime.datetime.fromisoformat(job['date_created'])
-                humanized_time = humanize.naturaltime(converted_time)
+            time_elapsed = "" if (job['status'] != RenderStatus.RUNNING.value and not end_time) else \
+                get_time_elapsed(start_time, end_time)
 
-                items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(engine_name),
-                         QTableWidgetItem(priority), QTableWidgetItem(display_status), QTableWidgetItem(time_elapsed),
-                         QTableWidgetItem(total_frames), QTableWidgetItem(humanized_time)]
+            name = job.get('name') or os.path.basename(job.get('input_path', ''))
+            engine_name = f"{job.get('engine', '')}-{job.get('engine_version')}"
+            priority = str(job.get('priority', ''))
+            total_frames = str(job.get('total_frames', ''))
+            converted_time = datetime.datetime.fromisoformat(job['date_created'])
+            humanized_time = humanize.naturaltime(converted_time)
 
-                for col, item in enumerate(items):
-                    self.job_list_view.setItem(row, col, item)
+            items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(engine_name),
+                     QTableWidgetItem(priority), QTableWidgetItem(display_status), QTableWidgetItem(time_elapsed),
+                     QTableWidgetItem(total_frames), QTableWidgetItem(humanized_time)]
+
+            for col, item in enumerate(items):
+                self.job_list_view.setItem(row, col, item)
 
     # -- Job Code -- #
     def job_picked(self):

src/utilities/server_helper.py

@@ -127,7 +127,7 @@ def distribute_server_work(start_frame, end_frame, available_servers, method='ev
         def fetch_benchmark(server):
             try:
                 benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}'
-                                         f'/api/cpu_benchmark').text
+                                         f'/api/cpu_benchmark', timeout=15).text
                 server['cpu_benchmark'] = benchmark
                 logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}')
             except requests.exceptions.RequestException as e: